phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 buildbot_url = ini.get("general", "buildbot_url")
  15
  16 # This is a sample buildmaster config file. It must be installed as
  17 # 'master.cfg' in your buildmaster's base directory.
  18
  19 # This is the dictionary that the buildmaster pays attention to. We also use
  20 # a shorter alias to save typing.
  21 c = BuildmasterConfig = {}
  22
  23 ####### BUILDSLAVES
  24
  25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  26 # a BuildSlave object, specifying a unique slave name and password.  The same
  27 # slave name and password must be configured on the slave.
  28 from buildbot.buildslave import BuildSlave
  29
  30 slave_port = 9990
  31
  32 if ini.has_option("general", "port"):
  33         slave_port = ini.getint("general", "port")
  34
  35 c['slaves'] = []
  36
  37 for section in ini.sections():
  38         if section.startswith("slave "):
  39                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  40                         name = ini.get(section, "name")
  41                         password = ini.get(section, "password")
  42                         max_builds = 1
  43                         if ini.has_option(section, "builds"):
  44                                 max_builds = ini.getint(section, "builds")
  45                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  46
  47 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  48 # This must match the value configured into the buildslaves (with their
  49 # --master option)
  50 c['slavePortnum'] = slave_port
  51
  52 # coalesce builds
  53 c['mergeRequests'] = True
  54
  55 ####### CHANGESOURCES
  56
  57 home_dir = os.path.abspath(ini.get("general", "homedir"))
  58
  59 rsync_bin_url = ini.get("rsync", "binary_url")
  60 rsync_bin_key = ini.get("rsync", "binary_password")
  61
  62 rsync_src_url = None
  63 rsync_src_key = None
  64
  65 if ini.has_option("rsync", "source_url"):
  66         rsync_src_url = ini.get("rsync", "source_url")
  67         rsync_src_key = ini.get("rsync", "source_password")
  68
  69 rsync_sdk_url = None
  70 rsync_sdk_key = None
  71 rsync_sdk_pat = "lede-sdk-*.tar.xz"
  72
  73 if ini.has_option("rsync", "sdk_url"):
  74         rsync_sdk_url = ini.get("rsync", "sdk_url")
  75
  76 if ini.has_option("rsync", "sdk_password"):
  77         rsync_sdk_key = ini.get("rsync", "sdk_password")
  78
  79 if ini.has_option("rsync", "sdk_pattern"):
  80         rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
  81
  82 gpg_keyid = None
  83 gpg_comment = "Unattended build signature"
  84 gpg_passfile = "/dev/null"
  85
  86 if ini.has_option("gpg", "keyid"):
  87         gpg_keyid = ini.get("gpg", "keyid")
  88
  89 if ini.has_option("gpg", "comment"):
  90         gpg_comment = ini.get("gpg", "comment")
  91
  92 if ini.has_option("gpg", "passfile"):
  93         gpg_passfile = ini.get("gpg", "passfile")
  94
  95
  96 # find arches
  97 arches = [ ]
  98 archnames = [ ]
  99
 100 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
 101         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 102
 103 while True:
 104         line = findarches.stdout.readline()
 105         if not line:
 106                 break
 107         at = line.strip().split()
 108         arches.append(at)
 109         archnames.append(at[0])
 110
 111
 112 # find feeds
 113 feeds = []
 114
 115 from buildbot.changes.gitpoller import GitPoller
 116 c['change_source'] = []
 117
 118 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 119         for line in f:
 120                 parts = line.strip().split()
 121                 if parts[0] == "src-git":
 122                         feeds.append(parts)
 123                         url = parts[2].strip().split(';')
 124                         branch = url[1] if len(url) > 1 else 'master'
 125                         c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 126
 127
 128 ####### SCHEDULERS
 129
 130 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 131 # case, just kick off a 'basebuild' build
 132
 133 from buildbot.schedulers.basic import SingleBranchScheduler
 134 from buildbot.schedulers.forcesched import ForceScheduler
 135 from buildbot.changes import filter
 136 c['schedulers'] = []
 137 c['schedulers'].append(SingleBranchScheduler(
 138         name="all",
 139         change_filter=filter.ChangeFilter(branch='master'),
 140         treeStableTimer=60,
 141         builderNames=archnames))
 142
 143 c['schedulers'].append(ForceScheduler(
 144         name="force",
 145         builderNames=archnames))
 146
 147 ####### BUILDERS
 148
 149 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 150 # what steps, and which slaves can execute them.  Note that any particular build will
 151 # only take place on one slave.
 152
 153 from buildbot.process.factory import BuildFactory
 154 from buildbot.steps.source import Git
 155 from buildbot.steps.shell import ShellCommand
 156 from buildbot.steps.shell import SetProperty
 157 from buildbot.steps.transfer import FileUpload
 158 from buildbot.steps.transfer import FileDownload
 159 from buildbot.steps.master import MasterShellCommand
 160 from buildbot.process.properties import WithProperties
 161
 162 c['builders'] = []
 163
 164 dlLock = locks.SlaveLock("slave_dl")
 165
 166 slaveNames = [ ]
 167
 168 for slave in c['slaves']:
 169         slaveNames.append(slave.slavename)
 170
 171 for arch in arches:
 172         ts = arch[1].split('/')
 173
 174         factory = BuildFactory()
 175
 176         # find number of cores
 177         factory.addStep(SetProperty(
 178                 name = "nproc",
 179                 property = "nproc",
 180                 description = "Finding number of CPUs",
 181                 command = ["nproc"]))
 182
 183         # prepare workspace
 184         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 185
 186         factory.addStep(ShellCommand(
 187                 name = "cleanold",
 188                 description = "Cleaning previous builds",
 189                 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 190                 haltOnFailure = True,
 191                 timeout = 2400))
 192
 193         factory.addStep(ShellCommand(
 194                 name = "cleanup",
 195                 description = "Cleaning work area",
 196                 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 197                 haltOnFailure = True,
 198                 timeout = 2400))
 199
 200         factory.addStep(ShellCommand(
 201                 name = "mksdkdir",
 202                 description = "Preparing SDK directory",
 203                 command = ["mkdir", "sdk"],
 204                 haltOnFailure = True))
 205
 206         factory.addStep(ShellCommand(
 207                 name = "downloadsdk",
 208                 description = "Downloading SDK archive",
 209                 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
 210                 env={'RSYNC_PASSWORD': rsync_sdk_key},
 211                 haltOnFailure = True,
 212                 logEnviron = False))
 213
 214         factory.addStep(ShellCommand(
 215                 name = "unpacksdk",
 216                 description = "Unpacking SDK archive",
 217                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
 218                 haltOnFailure = True))
 219
 220         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 221         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 222
 223         factory.addStep(ShellCommand(
 224                 name = "mkdldir",
 225                 description = "Preparing download directory",
 226                 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 227
 228         factory.addStep(ShellCommand(
 229                 name = "mkconf",
 230                 description = "Preparing SDK configuration",
 231                 workdir = "build/sdk",
 232                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 233
 234         factory.addStep(ShellCommand(
 235                 name = "updatefeeds",
 236                 description = "Updating feeds",
 237                 workdir = "build/sdk",
 238                 command = ["./scripts/feeds", "update"]))
 239
 240         factory.addStep(ShellCommand(
 241                 name = "installfeeds",
 242                 description = "Installing feeds",
 243                 workdir = "build/sdk",
 244                 command = ["./scripts/feeds", "install", "-a"]))
 245
 246         factory.addStep(ShellCommand(
 247                 name = "compile",
 248                 description = "Building packages",
 249                 workdir = "build/sdk",
 250                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 251
 252         if gpg_keyid is not None:
 253                 factory.addStep(MasterShellCommand(
 254                         name = "signprepare",
 255                         description = "Preparing temporary signing directory",
 256                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 257                         haltOnFailure = True
 258                 ))
 259
 260                 factory.addStep(ShellCommand(
 261                         name = "signpack",
 262                         description = "Packing files to sign",
 263                         workdir = "build/sdk",
 264                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 265                         haltOnFailure = True
 266                 ))
 267
 268                 factory.addStep(FileUpload(
 269                         slavesrc = "sdk/sign.tar.gz",
 270                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 271                         haltOnFailure = True
 272                 ))
 273
 274                 factory.addStep(MasterShellCommand(
 275                         name = "signfiles",
 276                         description = "Signing files",
 277                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 278                         haltOnFailure = True
 279                 ))
 280
 281                 factory.addStep(FileDownload(
 282                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 283                         slavedest = "sdk/sign.tar.gz",
 284                         haltOnFailure = True
 285                 ))
 286
 287                 factory.addStep(ShellCommand(
 288                         name = "signunpack",
 289                         description = "Unpacking signed files",
 290                         workdir = "build/sdk",
 291                         command = ["tar", "-xzf", "sign.tar.gz"],
 292                         haltOnFailure = True
 293                 ))
 294
 295         factory.addStep(ShellCommand(
 296                 name = "uploadprepare",
 297                 description = "Preparing package directory",
 298                 workdir = "build/sdk",
 299                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 300                 env={'RSYNC_PASSWORD': rsync_bin_key},
 301                 haltOnFailure = True,
 302                 logEnviron = False
 303         ))
 304
 305         factory.addStep(ShellCommand(
 306                 name = "packageupload",
 307                 description = "Uploading package files",
 308                 workdir = "build/sdk",
 309                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 310                 env={'RSYNC_PASSWORD': rsync_bin_key},
 311                 haltOnFailure = True,
 312                 logEnviron = False
 313         ))
 314
 315         factory.addStep(ShellCommand(
 316                 name = "logprepare",
 317                 description = "Preparing log directory",
 318                 workdir = "build/sdk",
 319                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 320                 env={'RSYNC_PASSWORD': rsync_bin_key},
 321                 haltOnFailure = True,
 322                 logEnviron = False
 323         ))
 324
 325         factory.addStep(ShellCommand(
 326                 name = "logfind",
 327                 description = "Finding failure logs",
 328                 workdir = "build/sdk/logs/package/feeds",
 329                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 330                 haltOnFailure = False
 331         ))
 332
 333         factory.addStep(ShellCommand(
 334                 name = "logcollect",
 335                 description = "Collecting failure logs",
 336                 workdir = "build/sdk",
 337                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 338                 haltOnFailure = False
 339         ))
 340
 341         factory.addStep(ShellCommand(
 342                 name = "logupload",
 343                 description = "Uploading failure logs",
 344                 workdir = "build/sdk",
 345                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 346                 env={'RSYNC_PASSWORD': rsync_bin_key},
 347                 haltOnFailure = False,
 348                 logEnviron = False
 349         ))
 350
 351         if rsync_src_url is not None:
 352                 factory.addStep(ShellCommand(
 353                         name = "sourceupload",
 354                         description = "Uploading source archives",
 355                         workdir = "build/sdk",
 356                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 357                         env={'RSYNC_PASSWORD': rsync_src_key},
 358                         haltOnFailure = False,
 359                         logEnviron = False
 360                 ))
 361
 362         from buildbot.config import BuilderConfig
 363
 364         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 365
 366
 367 ####### STATUS arches
 368
 369 # 'status' is a list of Status arches. The results of each build will be
 370 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 371 # including web pages, email senders, and IRC bots.
 372
 373 c['status'] = []
 374
 375 from buildbot.status import html
 376 from buildbot.status.web import authz, auth
 377
 378 if ini.has_option("status", "bind"):
 379         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 380                 authz_cfg=authz.Authz(
 381                         # change any of these to True to enable; see the manual for more
 382                         # options
 383                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 384                         gracefulShutdown = 'auth',
 385                         forceBuild = 'auth', # use this to test your slave once it is set up
 386                         forceAllBuilds = 'auth',
 387                         pingBuilder = False,
 388                         stopBuild = 'auth',
 389                         stopAllBuilds = 'auth',
 390                         cancelPendingBuild = 'auth',
 391                 )
 392                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 393         else:
 394                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 395
 396 ####### PROJECT IDENTITY
 397
 398 # the 'title' string will appear at the top of this buildbot
 399 # installation's html.WebStatus home page (linked to the
 400 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 401
 402 c['title'] = ini.get("general", "title")
 403 c['titleURL'] = ini.get("general", "title_url")
 404
 405 # the 'buildbotURL' string should point to the location where the buildbot's
 406 # internal web server (usually the html.WebStatus page) is visible. This
 407 # typically uses the port number set in the Waterfall 'status' entry, but
 408 # with an externally-visible host name which the buildbot cannot figure out
 409 # without some help.
 410
 411 c['buildbotURL'] = buildbot_url
 412
 413 ####### DB URL
 414
 415 c['db'] = {
 416         # This specifies what database buildbot uses to store its state.  You can leave
 417         # this at its default for all but the largest installations.
 418         'db_url' : "sqlite:///state.sqlite",
 419 }