phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 max_builds = dict()
  52 NetLocks = dict()
  53
  54 for section in ini.sections():
  55         if section.startswith("slave "):
  56                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  57                         sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False }
  58                         name = ini.get(section, "name")
  59                         password = ini.get(section, "password")
  60                         max_builds[name] = 1
  61                         if ini.has_option(section, "builds"):
  62                                 max_builds[name] = ini.getint(section, "builds")
  63                         if ini.has_option(section, "cleanup"):
  64                                 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
  65                         if ini.has_option(section, "dl_lock"):
  66                                 lockname = ini.get(section, "dl_lock")
  67                                 sl_props['dl_lock'] = lockname
  68                                 if lockname not in NetLocks:
  69                                         NetLocks[lockname] = locks.MasterLock(lockname)
  70                         if ini.has_option(section, "ul_lock"):
  71                                 lockname = ini.get(section, "dl_lock")
  72                                 sl_props['ul_lock'] = lockname
  73                                 if lockname not in NetLocks:
  74                                         NetLocks[lockname] = locks.MasterLock(lockname)
  75                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name], properties = sl_props))
  76
  77 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  78 # This must match the value configured into the buildslaves (with their
  79 # --master option)
  80 c['slavePortnum'] = slave_port
  81
  82 # coalesce builds
  83 c['mergeRequests'] = True
  84
  85 # Reduce amount of backlog data
  86 c['buildHorizon'] = 30
  87 c['logHorizon'] = 20
  88
  89 ####### CHANGESOURCES
  90
  91 home_dir = os.path.abspath(ini.get("general", "homedir"))
  92 tree_expire = 0
  93 other_builds = 0
  94 cc_version = None
  95
  96 cc_command = "gcc"
  97 cxx_command = "g++"
  98
  99 if ini.has_option("general", "expire"):
 100         tree_expire = ini.getint("general", "expire")
 101
 102 if ini.has_option("general", "other_builds"):
 103         other_builds = ini.getint("general", "other_builds")
 104
 105 if ini.has_option("general", "cc_version"):
 106         cc_version = ini.get("general", "cc_version").split()
 107         if len(cc_version) == 1:
 108                 cc_version = ["eq", cc_version[0]]
 109
 110 repo_url = ini.get("repo", "url")
 111 repo_branch = "master"
 112
 113 if ini.has_option("repo", "branch"):
 114         repo_branch = ini.get("repo", "branch")
 115
 116 rsync_bin_url = ini.get("rsync", "binary_url")
 117 rsync_bin_key = ini.get("rsync", "binary_password")
 118
 119 rsync_src_url = None
 120 rsync_src_key = None
 121
 122 if ini.has_option("rsync", "source_url"):
 123         rsync_src_url = ini.get("rsync", "source_url")
 124         rsync_src_key = ini.get("rsync", "source_password")
 125
 126 gpg_home = "~/.gnupg"
 127 gpg_keyid = None
 128 gpg_comment = "Unattended build signature"
 129 gpg_passfile = "/dev/null"
 130
 131 if ini.has_option("gpg", "home"):
 132         gpg_home = ini.get("gpg", "home")
 133
 134 if ini.has_option("gpg", "keyid"):
 135         gpg_keyid = ini.get("gpg", "keyid")
 136
 137 if ini.has_option("gpg", "comment"):
 138         gpg_comment = ini.get("gpg", "comment")
 139
 140 if ini.has_option("gpg", "passfile"):
 141         gpg_passfile = ini.get("gpg", "passfile")
 142
 143 enable_kmod_archive = True
 144
 145
 146 # find targets
 147 targets = [ ]
 148
 149 if not os.path.isdir(home_dir+'/source.git'):
 150         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 151 else:
 152         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 153
 154 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 155         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 156
 157 while True:
 158         line = findtargets.stdout.readline()
 159         if not line:
 160                 break
 161         ta = line.strip().split(' ')
 162         targets.append(ta[0])
 163
 164
 165 # the 'change_source' setting tells the buildmaster how it should find out
 166 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 167
 168 from buildbot.changes.gitpoller import GitPoller
 169 c['change_source'] = []
 170 c['change_source'].append(GitPoller(
 171         repo_url,
 172         workdir=home_dir+'/work.git', branch=repo_branch,
 173         pollinterval=300))
 174
 175 ####### SCHEDULERS
 176
 177 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 178 # case, just kick off a 'basebuild' build
 179
 180 from buildbot.schedulers.basic import SingleBranchScheduler
 181 from buildbot.schedulers.forcesched import ForceScheduler
 182 from buildbot.changes import filter
 183 c['schedulers'] = []
 184 c['schedulers'].append(SingleBranchScheduler(
 185         name="all",
 186         change_filter=filter.ChangeFilter(branch=repo_branch),
 187         treeStableTimer=60,
 188         builderNames=targets))
 189
 190 c['schedulers'].append(ForceScheduler(
 191         name="force",
 192         builderNames=targets))
 193
 194 ####### BUILDERS
 195
 196 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 197 # what steps, and which slaves can execute them.  Note that any particular build will
 198 # only take place on one slave.
 199
 200 from buildbot.process.factory import BuildFactory
 201 from buildbot.steps.source.git import Git
 202 from buildbot.steps.shell import ShellCommand
 203 from buildbot.steps.shell import SetProperty
 204 from buildbot.steps.shell import SetPropertyFromCommand
 205 from buildbot.steps.transfer import FileUpload
 206 from buildbot.steps.transfer import FileDownload
 207 from buildbot.steps.master import MasterShellCommand
 208 from buildbot.process.properties import WithProperties
 209 from buildbot.process import properties
 210
 211
 212 CleanTargetMap = [
 213         [ "tools",      "tools/clean"                   ],
 214         [ "chain",      "toolchain/clean"               ],
 215         [ "linux",      "target/linux/clean"    ],
 216         [ "dir",        "dirclean"                              ],
 217         [ "dist",       "distclean"                             ]
 218 ]
 219
 220 def IsMakeCleanRequested(pattern):
 221         def CheckCleanProperty(step):
 222                 val = step.getProperty("clean")
 223                 if val and re.match(pattern, val):
 224                         return True
 225                 else:
 226                         return False
 227
 228         return CheckCleanProperty
 229
 230 def IsCleanupRequested(step):
 231         do_cleanup = step.getProperty("do_cleanup")
 232         if do_cleanup:
 233                 return True
 234         else:
 235                 return False
 236
 237 def IsExpireRequested(step):
 238         return not IsCleanupRequested(step)
 239
 240 def IsGitFreshRequested(step):
 241         do_cleanup = step.getProperty("do_cleanup")
 242         if do_cleanup:
 243                 return True
 244         else:
 245                 return False
 246
 247 def IsGitCleanRequested(step):
 248         return not IsGitFreshRequested(step)
 249
 250 def IsTaggingRequested(step):
 251         val = step.getProperty("tag")
 252         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 253                 return True
 254         else:
 255                 return False
 256
 257 def IsNoTaggingRequested(step):
 258         return not IsTaggingRequested(step)
 259
 260 def IsNoMasterBuild(step):
 261         return repo_branch != "master"
 262
 263 def IsCleanupConfigured(step):
 264         do_cleanup = step.getProperty("do_cleanup")
 265         if do_cleanup:
 266                 return do_cleanup > 0
 267         else:
 268                 return False
 269
 270 def GetBaseVersion(props):
 271         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 272                 return repo_branch.split('-')[1]
 273         else:
 274                 return "master"
 275
 276 def GetVersionPrefix(props):
 277         basever = GetBaseVersion(props)
 278         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 279                 return "%s/" % props["tag"]
 280         elif basever != "master":
 281                 return "%s-SNAPSHOT/" % basever
 282         else:
 283                 return ""
 284
 285 def GetNumJobs(props):
 286         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 287                 return (int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds))
 288         else:
 289                 return 1
 290
 291 def GetCC(props):
 292         if props.hasProperty("cc_command"):
 293                 return props["cc_command"]
 294         else:
 295                 return "gcc"
 296
 297 def GetCXX(props):
 298         if props.hasProperty("cxx_command"):
 299                 return props["cxx_command"]
 300         else:
 301                 return "g++"
 302
 303 def GetCwd(props):
 304         if props.hasProperty("builddir"):
 305                 return props["builddir"]
 306         elif props.hasProperty("workdir"):
 307                 return props["workdir"]
 308         else:
 309                 return "/"
 310
 311 def GetCCache(props):
 312         if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
 313                 return props["ccache_command"] + " "
 314         else:
 315                 return ""
 316
 317 def GetNextBuild(builder, requests):
 318         for r in requests:
 319                 if r.properties and r.properties.hasProperty("tag"):
 320                         return r
 321         return requests[0]
 322
 323 def MakeEnv(overrides=None, tryccache=False):
 324         if tryccache:
 325                 envcc = Interpolate("%(kw:ccache)s%(kw:cc)s", ccache=GetCCache, cc=GetCC)
 326                 envcxx = Interpolate("%(kw:ccache)s%(kw:cxx)s", ccache=GetCCache, cxx=GetCXX)
 327         else:
 328                 envcc = Interpolate("%(kw:cc)s", cc=GetCC)
 329                 envcxx = Interpolate("%(kw:cxx)s", cxx=GetCXX)
 330         env = {
 331                 'CC': envcc,
 332                 'CXX': envcxx,
 333                 'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)
 334         }
 335         if overrides is not None:
 336                 env.update(overrides)
 337         return env
 338
 339 @properties.renderer
 340 def NetLockDl(props):
 341         lock = None
 342         if props.hasProperty("dl_lock"):
 343                 lock = NetLocks[props["dl_lock"]]
 344         if lock is not None:
 345                 return [lock.access('exclusive')]
 346         else:
 347                 return []
 348
 349 @properties.renderer
 350 def NetLockUl(props):
 351         lock = None
 352         if props.hasProperty("ul_lock"):
 353                 lock = NetLocks[props["ul_lock"]]
 354         if lock is not None:
 355                 return [lock.access('exclusive')]
 356         else:
 357                 return []
 358
 359 c['builders'] = []
 360
 361 dlLock = locks.SlaveLock("slave_dl")
 362
 363 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 364         checkBuiltin() {
 365                 local symbol op path file;
 366                 for file in $CHANGED_FILES; do
 367                         case "$file" in
 368                                 package/*/*) : ;;
 369                                 *) return 0 ;;
 370                         esac;
 371                 done;
 372                 while read symbol op path; do
 373                         case "$symbol" in package-*)
 374                                 symbol="${symbol##*(}";
 375                                 symbol="${symbol%)}";
 376                                 for file in $CHANGED_FILES; do
 377                                         case "$file" in "package/$path/"*)
 378                                                 grep -qsx "$symbol=y" .config && return 0
 379                                         ;; esac;
 380                                 done;
 381                         esac;
 382                 done < tmp/.packagedeps;
 383                 return 1;
 384         }
 385 """).strip()
 386
 387
 388 class IfBuiltinShellCommand(ShellCommand):
 389         def _quote(self, str):
 390                 if re.search("[^a-zA-Z0-9/_.-]", str):
 391                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 392                 return str
 393
 394         def setCommand(self, command):
 395                 if not isinstance(command, (str, unicode)):
 396                         command = ' '.join(map(self._quote, command))
 397                 self.command = [
 398                         '/bin/sh', '-c',
 399                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 400                 ]
 401
 402         def setupEnvironment(self, cmd):
 403                 slaveEnv = self.slaveEnvironment
 404                 if slaveEnv is None:
 405                         slaveEnv = { }
 406                 changedFiles = { }
 407                 for request in self.build.requests:
 408                         for source in request.sources:
 409                                 for change in source.changes:
 410                                         for file in change.files:
 411                                                 changedFiles[file] = True
 412                 fullSlaveEnv = slaveEnv.copy()
 413                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 414                 cmd.args['env'] = fullSlaveEnv
 415
 416 slaveNames = [ ]
 417
 418 for slave in c['slaves']:
 419         slaveNames.append(slave.slavename)
 420
 421 for target in targets:
 422         ts = target.split('/')
 423
 424         factory = BuildFactory()
 425
 426         # find number of cores
 427         factory.addStep(SetProperty(
 428                 name = "nproc",
 429                 property = "nproc",
 430                 description = "Finding number of CPUs",
 431                 command = ["nproc"]))
 432
 433         # find gcc and g++ compilers
 434         if cc_version is not None:
 435                 factory.addStep(FileDownload(
 436                         mastersrc = "findbin.pl",
 437                         slavedest = "../findbin.pl",
 438                         mode = 0755))
 439
 440                 factory.addStep(SetProperty(
 441                         name = "gcc",
 442                         property = "cc_command",
 443                         description = "Finding gcc command",
 444                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 445                         haltOnFailure = True))
 446
 447                 factory.addStep(SetProperty(
 448                         name = "g++",
 449                         property = "cxx_command",
 450                         description = "Finding g++ command",
 451                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 452                         haltOnFailure = True))
 453
 454         # see if ccache is available
 455         factory.addStep(SetPropertyFromCommand(
 456                 property = "ccache_command",
 457                 command = ["which", "ccache"],
 458                 description = "Testing for ccache command",
 459                 haltOnFailure = False,
 460                 flunkOnFailure = False,
 461                 warnOnFailure = False,
 462         ))
 463
 464         # expire tree if needed
 465         if tree_expire > 0:
 466                 factory.addStep(FileDownload(
 467                         doStepIf = IsExpireRequested,
 468                         mastersrc = "expire.sh",
 469                         slavedest = "../expire.sh",
 470                         mode = 0755))
 471
 472                 factory.addStep(ShellCommand(
 473                         name = "expire",
 474                         description = "Checking for build tree expiry",
 475                         command = ["./expire.sh", str(tree_expire)],
 476                         workdir = ".",
 477                         haltOnFailure = True,
 478                         doStepIf = IsExpireRequested,
 479                         timeout = 2400))
 480
 481         # cleanup.sh if needed
 482         factory.addStep(FileDownload(
 483                 mastersrc = "cleanup.sh",
 484                 slavedest = "../cleanup.sh",
 485                 mode = 0755,
 486                 doStepIf = IsCleanupRequested))
 487
 488         factory.addStep(ShellCommand(
 489                 name = "cleanold",
 490                 description = "Cleaning previous builds",
 491                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 492                 workdir = ".",
 493                 haltOnFailure = True,
 494                 doStepIf = IsCleanupRequested,
 495                 timeout = 2400))
 496
 497         factory.addStep(ShellCommand(
 498                 name = "cleanup",
 499                 description = "Cleaning work area",
 500                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 501                 workdir = ".",
 502                 haltOnFailure = True,
 503                 doStepIf = IsCleanupRequested,
 504                 timeout = 2400))
 505
 506         # user-requested clean targets
 507         for tuple in CleanTargetMap:
 508                 factory.addStep(ShellCommand(
 509                         name = tuple[1],
 510                         description = 'User-requested "make %s"' % tuple[1],
 511                         command = ["make", tuple[1], "V=s"],
 512                         env = MakeEnv(),
 513                         doStepIf = IsMakeCleanRequested(tuple[0])
 514                 ))
 515
 516         # check out the source
 517         # Git() runs:
 518           # if repo doesn't exist: 'git clone repourl'
 519           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
 520           # 'git fetch -t repourl branch; git reset --hard revision'
 521         # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
 522         # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
 523         factory.addStep(Git(
 524                 repourl = repo_url,
 525                 branch = repo_branch,
 526                 mode = 'full',
 527                 method = 'clean',
 528                 locks = NetLockDl,
 529                 haltOnFailure = True,
 530                 doStepIf = IsGitCleanRequested,
 531         ))
 532
 533         factory.addStep(Git(
 534                 repourl = repo_url,
 535                 branch = repo_branch,
 536                 mode = 'full',
 537                 method = 'fresh',
 538                 locks = NetLockDl,
 539                 haltOnFailure = True,
 540                 doStepIf = IsGitFreshRequested,
 541         ))
 542
 543         # update remote refs
 544         factory.addStep(ShellCommand(
 545                 name = "fetchrefs",
 546                 description = "Fetching Git remote refs",
 547                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 548                 haltOnFailure = True
 549         ))
 550
 551         # switch to tag
 552         factory.addStep(ShellCommand(
 553                 name = "switchtag",
 554                 description = "Checking out Git tag",
 555                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 556                 haltOnFailure = True,
 557                 doStepIf = IsTaggingRequested
 558         ))
 559
 560         factory.addStep(ShellCommand(
 561                 name = "rmtmp",
 562                 description = "Remove tmp folder",
 563                 command=["rm", "-rf", "tmp/"]))
 564
 565         # feed
 566 #       factory.addStep(ShellCommand(
 567 #               name = "feedsconf",
 568 #               description = "Copy the feeds.conf",
 569 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 570
 571         # feed
 572         factory.addStep(ShellCommand(
 573                 name = "rmfeedlinks",
 574                 description = "Remove feed symlinks",
 575                 command=["rm", "-rf", "package/feeds/"]))
 576
 577         # feed
 578         factory.addStep(ShellCommand(
 579                 name = "updatefeeds",
 580                 description = "Updating feeds",
 581                 command=["./scripts/feeds", "update"],
 582                 env = MakeEnv(),
 583                 locks = NetLockDl,
 584         ))
 585
 586         # feed
 587         factory.addStep(ShellCommand(
 588                 name = "installfeeds",
 589                 description = "Installing feeds",
 590                 command=["./scripts/feeds", "install", "-a"],
 591                 env = MakeEnv()))
 592
 593         # seed config
 594         factory.addStep(FileDownload(
 595                 mastersrc = "config.seed",
 596                 slavedest = ".config",
 597                 mode = 0644
 598         ))
 599
 600         # configure
 601         factory.addStep(ShellCommand(
 602                 name = "newconfig",
 603                 description = "Seeding .config",
 604                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 605         ))
 606
 607         factory.addStep(ShellCommand(
 608                 name = "delbin",
 609                 description = "Removing output directory",
 610                 command = ["rm", "-rf", "bin/"]
 611         ))
 612
 613         factory.addStep(ShellCommand(
 614                 name = "defconfig",
 615                 description = "Populating .config",
 616                 command = ["make", "defconfig"],
 617                 env = MakeEnv()
 618         ))
 619
 620         # check arch
 621         factory.addStep(ShellCommand(
 622                 name = "checkarch",
 623                 description = "Checking architecture",
 624                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 625                 logEnviron = False,
 626                 want_stdout = False,
 627                 want_stderr = False,
 628                 haltOnFailure = True
 629         ))
 630
 631         # find libc suffix
 632         factory.addStep(SetProperty(
 633                 name = "libc",
 634                 property = "libc",
 635                 description = "Finding libc suffix",
 636                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 637
 638         # ccache helper
 639         factory.addStep(FileDownload(
 640                 mastersrc = "ccache.sh",
 641                 slavedest = "../ccache.sh",
 642                 mode = 0755
 643         ))
 644
 645         # ccache prepare
 646         factory.addStep(ShellCommand(
 647                 name = "prepccache",
 648                 description = "Preparing ccache",
 649                 command = ["../ccache.sh"]
 650         ))
 651
 652         # install build key
 653         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 654         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 655
 656         # prepare dl
 657         factory.addStep(ShellCommand(
 658                 name = "dldir",
 659                 description = "Preparing dl/",
 660                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 661                 logEnviron = False,
 662                 want_stdout = False
 663         ))
 664
 665         # prepare tar
 666         factory.addStep(ShellCommand(
 667                 name = "dltar",
 668                 description = "Building and installing GNU tar",
 669                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 670                 env = MakeEnv(tryccache=True),
 671                 haltOnFailure = True
 672         ))
 673
 674         # populate dl
 675         factory.addStep(ShellCommand(
 676                 name = "dlrun",
 677                 description = "Populating dl/",
 678                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 679                 env = MakeEnv(),
 680                 logEnviron = False,
 681                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 682         ))
 683
 684         factory.addStep(ShellCommand(
 685                 name = "cleanbase",
 686                 description = "Cleaning base-files",
 687                 command=["make", "package/base-files/clean", "V=s"]
 688         ))
 689
 690         # build
 691         factory.addStep(ShellCommand(
 692                 name = "tools",
 693                 description = "Building and installing tools",
 694                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 695                 env = MakeEnv(tryccache=True),
 696                 haltOnFailure = True
 697         ))
 698
 699         factory.addStep(ShellCommand(
 700                 name = "toolchain",
 701                 description = "Building and installing toolchain",
 702                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 703                 env = MakeEnv(),
 704                 haltOnFailure = True
 705         ))
 706
 707         factory.addStep(ShellCommand(
 708                 name = "kmods",
 709                 description = "Building kmods",
 710                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 711                 env = MakeEnv(),
 712                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 713                 haltOnFailure = True
 714         ))
 715
 716         # find kernel version
 717         factory.addStep(SetProperty(
 718                 name = "kernelversion",
 719                 property = "kernelversion",
 720                 description = "Finding the effective Kernel version",
 721                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 722                 env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 723         ))
 724
 725         factory.addStep(ShellCommand(
 726                 name = "pkgclean",
 727                 description = "Cleaning up package build",
 728                 command=["make", "package/cleanup", "V=s"]
 729         ))
 730
 731         factory.addStep(ShellCommand(
 732                 name = "pkgbuild",
 733                 description = "Building packages",
 734                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 735                 env = MakeEnv(),
 736                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 737                 haltOnFailure = True
 738         ))
 739
 740         # factory.addStep(IfBuiltinShellCommand(
 741         factory.addStep(ShellCommand(
 742                 name = "pkginstall",
 743                 description = "Installing packages",
 744                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 745                 env = MakeEnv(),
 746                 haltOnFailure = True
 747         ))
 748
 749         factory.addStep(ShellCommand(
 750                 name = "pkgindex",
 751                 description = "Indexing packages",
 752                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 753                 env = MakeEnv(),
 754                 haltOnFailure = True
 755         ))
 756
 757         if enable_kmod_archive:
 758                 factory.addStep(ShellCommand(
 759                         name = "kmoddir",
 760                         description = "Creating kmod directory",
 761                         command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))],
 762                         haltOnFailure = True
 763                 ))
 764
 765                 factory.addStep(ShellCommand(
 766                         name = "kmodprepare",
 767                         description = "Preparing kmod archive",
 768                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 769                                  WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])),
 770                                  WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 771                         haltOnFailure = True
 772                 ))
 773
 774                 factory.addStep(ShellCommand(
 775                         name = "kmodindex",
 776                         description = "Indexing kmod archive",
 777                         command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s",
 778                                  WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 779                         env = MakeEnv(),
 780                         haltOnFailure = True
 781                 ))
 782
 783                 # find rootfs staging directory
 784                 factory.addStep(SetProperty(
 785                         name = "stageroot",
 786                         property = "stageroot",
 787                         description = "Finding the rootfs staging directory",
 788                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 789                         env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 790                 ))
 791
 792                 factory.addStep(ShellCommand(
 793                         name = "filesdir",
 794                         description = "Creating file overlay directory",
 795                         command=["mkdir", "-p", "files/etc/opkg"],
 796                         haltOnFailure = True
 797                 ))
 798
 799                 factory.addStep(ShellCommand(
 800                         name = "kmodconfig",
 801                         description = "Embedding kmod repository configuration",
 802                         command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " +
 803                                                "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 804                         haltOnFailure = True
 805                 ))
 806
 807         #factory.addStep(IfBuiltinShellCommand(
 808         factory.addStep(ShellCommand(
 809                 name = "images",
 810                 description = "Building and installing images",
 811                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 812                 env = MakeEnv(),
 813                 haltOnFailure = True
 814         ))
 815
 816         factory.addStep(ShellCommand(
 817                 name = "diffconfig",
 818                 description = "Generating config.seed",
 819                 command=["make", "-j1", "diffconfig", "V=s"],
 820                 env = MakeEnv(),
 821                 haltOnFailure = True
 822         ))
 823
 824         factory.addStep(ShellCommand(
 825                 name = "checksums",
 826                 description = "Calculating checksums",
 827                 command=["make", "-j1", "checksum", "V=s"],
 828                 env = MakeEnv(),
 829                 haltOnFailure = True
 830         ))
 831
 832         # sign
 833         if gpg_keyid is not None:
 834                 factory.addStep(MasterShellCommand(
 835                         name = "signprepare",
 836                         description = "Preparing temporary signing directory",
 837                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 838                         haltOnFailure = True
 839                 ))
 840
 841                 factory.addStep(ShellCommand(
 842                         name = "signpack",
 843                         description = "Packing files to sign",
 844                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])),
 845                         haltOnFailure = True
 846                 ))
 847
 848                 factory.addStep(FileUpload(
 849                         slavesrc = "sign.tar.gz",
 850                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 851                         haltOnFailure = True
 852                 ))
 853
 854                 factory.addStep(MasterShellCommand(
 855                         name = "signfiles",
 856                         description = "Signing files",
 857                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 858                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 859                         haltOnFailure = True
 860                 ))
 861
 862                 factory.addStep(FileDownload(
 863                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 864                         slavedest = "sign.tar.gz",
 865                         haltOnFailure = True
 866                 ))
 867
 868                 factory.addStep(ShellCommand(
 869                         name = "signunpack",
 870                         description = "Unpacking signed files",
 871                         command = ["tar", "-xzf", "sign.tar.gz"],
 872                         haltOnFailure = True
 873                 ))
 874
 875         # upload
 876         factory.addStep(ShellCommand(
 877                 name = "dirprepare",
 878                 description = "Preparing upload directory structure",
 879                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 880                 haltOnFailure = True
 881         ))
 882
 883         factory.addStep(ShellCommand(
 884                 name = "linkprepare",
 885                 description = "Preparing repository symlink",
 886                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 887                 doStepIf = IsNoMasterBuild,
 888                 haltOnFailure = True
 889         ))
 890
 891         if enable_kmod_archive:
 892                 factory.addStep(ShellCommand(
 893                         name = "kmoddirprepare",
 894                         description = "Preparing kmod archive upload directory",
 895                         command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 896                         haltOnFailure = True
 897                 ))
 898
 899         factory.addStep(ShellCommand(
 900                 name = "dirupload",
 901                 description = "Uploading directory structure",
 902                 command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 903                 env={'RSYNC_PASSWORD': rsync_bin_key},
 904                 haltOnFailure = True,
 905                 logEnviron = False,
 906                 locks = NetLockUl,
 907         ))
 908
 909         factory.addStep(FileDownload(
 910                         mastersrc = "rsync.sh",
 911                         slavedest = "../rsync.sh",
 912                         mode = 0755))
 913
 914         factory.addStep(ShellCommand(
 915                 name = "targetupload",
 916                 description = "Uploading target files",
 917                 command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 918                          "-a", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 919                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 920                 env={'RSYNC_PASSWORD': rsync_bin_key},
 921                 haltOnFailure = True,
 922                 logEnviron = False,
 923                 locks = NetLockUl,
 924         ))
 925
 926         if enable_kmod_archive:
 927                 factory.addStep(ShellCommand(
 928                         name = "kmodupload",
 929                         description = "Uploading kmod archive",
 930                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 931                                  "-a", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
 932                                  WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 933                         env={'RSYNC_PASSWORD': rsync_bin_key},
 934                         haltOnFailure = True,
 935                         logEnviron = False,
 936                         locks = NetLockUl,
 937                 ))
 938
 939         if rsync_src_url is not None:
 940                 factory.addStep(ShellCommand(
 941                         name = "sourceupload",
 942                         description = "Uploading source archives",
 943                         command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
 944                                  WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-a", "dl/", "%s/" %(rsync_src_url)],
 945                         env={'RSYNC_PASSWORD': rsync_src_key},
 946                         haltOnFailure = True,
 947                         logEnviron = False,
 948                         locks = NetLockUl,
 949                 ))
 950
 951         if False:
 952                 factory.addStep(ShellCommand(
 953                         name = "packageupload",
 954                         description = "Uploading package files",
 955                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 956                         env={'RSYNC_PASSWORD': rsync_bin_key},
 957                         haltOnFailure = False,
 958                         logEnviron = False,
 959                         locks = NetLockUl,
 960                 ))
 961
 962         # logs
 963         if False:
 964                 factory.addStep(ShellCommand(
 965                         name = "upload",
 966                         description = "Uploading logs",
 967                         command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 968                         env={'RSYNC_PASSWORD': rsync_bin_key},
 969                         haltOnFailure = False,
 970                         alwaysRun = True,
 971                         logEnviron = False,
 972                         locks = NetLockUl,
 973                 ))
 974
 975         factory.addStep(ShellCommand(
 976                 name = "df",
 977                 description = "Reporting disk usage",
 978                 command=["df", "-h", "."],
 979                 env={'LC_ALL': 'C'},
 980                 haltOnFailure = False,
 981                 alwaysRun = True
 982         ))
 983
 984         factory.addStep(ShellCommand(
 985                 name = "ccachestat",
 986                 description = "Reporting ccache stats",
 987                 command=["ccache", "-s"],
 988                 env = MakeEnv(overrides={ 'PATH': ["./staging_dir/host/bin", "${PATH}"] }),
 989                 want_stderr = False,
 990                 haltOnFailure = False,
 991                 flunkOnFailure = False,
 992                 warnOnFailure = False,
 993                 alwaysRun = True,
 994         ))
 995
 996         from buildbot.config import BuilderConfig
 997
 998         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 999
1000
1001 ####### STATUS TARGETS
1002
1003 # 'status' is a list of Status Targets. The results of each build will be
1004 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1005 # including web pages, email senders, and IRC bots.
1006
1007 c['status'] = []
1008
1009 from buildbot.status import html
1010 from buildbot.status.web import authz, auth
1011
1012 if ini.has_option("status", "bind"):
1013         if ini.has_option("status", "user") and ini.has_option("status", "password"):
1014                 authz_cfg=authz.Authz(
1015                         # change any of these to True to enable; see the manual for more
1016                         # options
1017                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
1018                         gracefulShutdown = 'auth',
1019                         forceBuild = 'auth', # use this to test your slave once it is set up
1020                         forceAllBuilds = 'auth',
1021                         pingBuilder = False,
1022                         stopBuild = 'auth',
1023                         stopAllBuilds = 'auth',
1024                         cancelPendingBuild = 'auth',
1025                 )
1026                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
1027         else:
1028                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
1029
1030
1031 from buildbot.status import words
1032
1033 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1034         irc_host = ini.get("irc", "host")
1035         irc_port = 6667
1036         irc_chan = ini.get("irc", "channel")
1037         irc_nick = ini.get("irc", "nickname")
1038         irc_pass = None
1039
1040         if ini.has_option("irc", "port"):
1041                 irc_port = ini.getint("irc", "port")
1042
1043         if ini.has_option("irc", "password"):
1044                 irc_pass = ini.get("irc", "password")
1045
1046         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
1047                         channels = [{ "channel": irc_chan }],
1048                         notify_events = {
1049                           'exception': 1,
1050                           'successToFailure': 1,
1051                           'failureToSuccess': 1
1052                         }
1053         )
1054
1055         c['status'].append(irc)
1056
1057 ####### DB URL
1058
1059 c['db'] = {
1060         # This specifies what database buildbot uses to store its state.  You can leave
1061         # this at its default for all but the largest installations.
1062         'db_url' : "sqlite:///state.sqlite",
1063 }