<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[146776] contrib</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="https://trac.macports.org/changeset/146776">146776</a></dd>
<dt>Author</dt> <dd>mojca@macports.org</dd>
<dt>Date</dt> <dd>2016-03-17 02:43:06 -0700 (Thu, 17 Mar 2016)</dd>
</dl>
<h3>Log Message</h3>
<pre>buildbot improvements</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#contribbuildbottestmastercfg">contrib/buildbot-test/master.cfg</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li>contrib/buildbot-test/</li>
<li><a href="#contribbuildbottestconfigjsonsample">contrib/buildbot-test/config.json.sample</a></li>
<li><a href="#contribbuildbottestslavesjsonsample">contrib/buildbot-test/slaves.json.sample</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="contribbuildbottestconfigjsonsample"></a>
<div class="addfile"><h4>Added: contrib/buildbot-test/config.json.sample (0 => 146776)</h4>
<pre class="diff"><span>
<span class="info">--- contrib/buildbot-test/config.json.sample (rev 0)
+++ contrib/buildbot-test/config.json.sample 2016-03-17 09:43:06 UTC (rev 146776)
</span><span class="lines">@@ -0,0 +1,9 @@
</span><ins>+{
+ "production": 0,
+ "privkey": "/var/keys/macports-privkey.pem",
+ "slaveport": 9989,
+ "httpport": 8010,
+ "buildboturl": "http://localhost:8010/",
+ "htpasswdfile": "htpasswd",
+ "mpbbsvnurl": "https://svn.macports.org/repository/macports/contrib/mp-buildbot"
+}
</ins></span></pre></div>
<a id="contribbuildbottestmastercfg"></a>
<div class="modfile"><h4>Modified: contrib/buildbot-test/master.cfg (146766 => 146776)</h4>
<pre class="diff"><span>
<span class="info">--- contrib/buildbot/master.cfg 2016-03-17 04:17:40 UTC (rev 146766)
+++ contrib/buildbot-test/master.cfg 2016-03-17 09:43:06 UTC (rev 146776)
</span><span class="lines">@@ -1,5 +1,6 @@
</span><span class="cx"> # -*- python -*-
</span><span class="cx"> # ex: set syntax=python:
</span><ins>+# vim:ft=python
</ins><span class="cx">
</span><span class="cx"> import json
</span><span class="cx"> import os
</span><span class="lines">@@ -21,18 +22,27 @@
</span><span class="cx">
</span><span class="cx"> production = False
</span><span class="cx"> privkey = ''
</span><ins>+slaveport = 9989
+httpport = 8010
+buildboturl = "http://localhost:8010/"
+htpasswdfile = "htpasswd"
+mpbbsvnurl = "https://svn.macports.org/repository/macports/contrib/mp-buildbot"
+
</ins><span class="cx"> if os.path.exists(_path('config.json')):
</span><span class="cx"> with open(_path('config.json')) as f:
</span><span class="cx"> configdata = json.load(f)
</span><del>- production = configdata['production']
- privkey = configdata['privkey']
</del><ins>+ locals = locals()
+ for key in ["production", "privkey", "slaveport", "httpport", "buildboturl", "htpasswdfile", "mpbbsvnurl"]:
+ if configdata[key]:
+ locals[key] = configdata[key]
</ins><span class="cx">
</span><span class="cx"> # platforms we are building for
</span><span class="cx"> if production:
</span><span class="cx"> # this list should get longer in future
</span><del>- build_platforms = ["snowleopard-x86_64", "lion-x86_64", "mtln-x86_64"]
</del><ins>+ #build_platforms = ["snowleopard-x86_64", "lion-x86_64", "mtln-x86_64", "mavericks-x86_64", "yosemite-x86_64", "elcapitan-x86_64", "oelinux-6.3-x86_64"]
+ build_platforms = []
</ins><span class="cx"> else:
</span><del>- build_platforms = ["snowleopard-x86_64"]
</del><ins>+ build_platforms = ["10.6-x86_64","10.9-x86_64"]
</ins><span class="cx">
</span><span class="cx"> # Allow spaces and tabs in property values
</span><span class="cx"> import re
</span><span class="lines">@@ -49,19 +59,17 @@
</span><span class="cx">
</span><span class="cx"> c['slaves'] = []
</span><span class="cx"> slavedata = {}
</span><del>-if production:
- with open(_path('slaves.json')) as f:
- slavedata = json.load(f)
- for slave, pwd in slavedata.items():
- c['slaves'].append(BuildSlave(slave, pwd))
-else:
- c['slaves'] = [BuildSlave("snowleopard-x86_64", "pass")]
</del><span class="cx">
</span><ins>+with open(_path('slaves.json')) as f:
+ slavedata = json.load(f)
+for slave, pwd in slavedata.items():
+ c['slaves'].append(BuildSlave(slave, pwd))
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
</span><span class="cx"> # This must match the value configured into the buildslaves (with their
</span><span class="cx"> # --master option)
</span><del>-c['slavePortnum'] = 17000
</del><ins>+c['slavePortnum'] = slaveport
</ins><span class="cx">
</span><span class="cx"> ####### CHANGESOURCES
</span><span class="cx">
</span><span class="lines">@@ -74,16 +82,20 @@
</span><span class="cx"> if production:
</span><span class="cx"> from buildbot.changes.pb import PBChangeSource
</span><span class="cx"> sourcedata = []
</span><del>- with open(_path('source.json')) as f:
- sourcedata = json.load(f)
- c['change_source'] = PBChangeSource(user=sourcedata[0], passwd=sourcedata[1], port=sourcedata[2])
</del><ins>+# with open(_path('source.json')) as f:
+# sourcedata = json.load(f)
+# c['change_source'] = PBChangeSource(user=sourcedata[0], passwd=sourcedata[1], port=sourcedata[2])
</ins><span class="cx"> else:
</span><span class="cx"> from buildbot.changes.svnpoller import SVNPoller
</span><del>- c['change_source'] = [SVNPoller(
- 'https://svn.macports.org/repository/macports/trunk',
- svnbin='/opt/local/bin/svn',
- pollinterval=300),
- ]
</del><ins>+ c['change_source'] = (SVNPoller(
+ svnurl='https://svn.macports.org/repository/macports/trunk',
+ #'https://svn.macports.org/repository/macports/trunk',
+ #svnbin='/opt/local/bin/svn',
+ pollinterval=300,
+ category='macports',
+ project='ports',
+ ),
+ )
</ins><span class="cx">
</span><span class="cx"> ####### SCHEDULERS
</span><span class="cx">
</span><span class="lines">@@ -107,11 +119,15 @@
</span><span class="cx"> portsfilter = ChangeFilter(filter_fn=change_has_ports)
</span><span class="cx"> basefilter = ChangeFilter(filter_fn=change_has_base)
</span><span class="cx">
</span><del>-base_buildernames = ["buildbase-"+plat for plat in build_platforms]
-ports_buildernames = ["buildports-"+plat for plat in build_platforms if 'linux' not in plat]
</del><ins>+base_buildernames = ["base-"+plat for plat in build_platforms]
+portwatcher_buildernames = ["portwatcher-"+plat for plat in build_platforms if 'linux' not in plat]
+portbuilder_buildernames = ["portbuilder-"+plat for plat in build_platforms if 'linux' not in plat]
+portbuilder_triggerables = ["triggerable-"+plat for plat in portbuilder_buildernames]
</ins><span class="cx">
</span><span class="cx"> from buildbot.schedulers.basic import SingleBranchScheduler
</span><span class="cx"> from buildbot.schedulers.forcesched import ForceScheduler
</span><ins>+from buildbot.schedulers.triggerable import Triggerable
+
</ins><span class="cx"> c['schedulers'] = [SingleBranchScheduler(
</span><span class="cx"> name="base",
</span><span class="cx"> treeStableTimer=None,
</span><span class="lines">@@ -121,148 +137,186 @@
</span><span class="cx"> name="ports",
</span><span class="cx"> treeStableTimer=None,
</span><span class="cx"> change_filter = portsfilter,
</span><del>- builderNames=ports_buildernames),
</del><ins>+ builderNames=portwatcher_buildernames),
</ins><span class="cx"> ForceScheduler(
</span><span class="cx"> name="force",
</span><del>- builderNames=ports_buildernames+base_buildernames)
</del><ins>+ builderNames=portwatcher_buildernames+portbuilder_buildernames+base_buildernames)
</ins><span class="cx"> ]
</span><span class="cx">
</span><ins>+for plat in portbuilder_buildernames:
+ c['schedulers'].append(Triggerable( name="triggerable-"+plat, builderNames=[plat] ))
+
</ins><span class="cx"> ####### BUILDERS
</span><span class="cx">
</span><del>-#c['mergeRequests'] = True
</del><ins>+# WARNING: mergeRequests has to be False or Triggerable builds will not be scheduled correctly!
+c['mergeRequests'] = False
</ins><span class="cx">
</span><span class="cx"> # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
</span><span class="cx"> # what steps, and which slaves can execute them. Note that any particular build will
</span><span class="cx"> # only take place on one slave.
</span><span class="cx">
</span><span class="cx"> from buildbot.process.factory import BuildFactory, GNUAutoconf
</span><del>-from buildbot.process.properties import WithProperties
-from buildbot.steps.source import SVN
-from buildbot.steps.shell import ShellCommand, Compile, Configure
</del><ins>+from buildbot.process.properties import WithProperties, Interpolate
+from buildbot.steps.source.svn import SVN
+from buildbot.steps.shell import ShellCommand, Compile, Configure, SetPropertyFromCommand
</ins><span class="cx">
</span><span class="cx"> base_factory = BuildFactory()
</span><del>-base_factory.addStep(SVN(baseURL='https://svn.macports.org/repository/macports/%%BRANCH%%/base',
- mode="copy", defaultBranch="trunk"))
-maybe_sudo="""
-if [[ `id -u` -eq 0 ]]; then
- [[ -n "$SUDO_USER" ]] || SUDO_USER=buildbot
- CMD_PREFIX="sudo -u $SUDO_USER"
- INSTALL_USER=$SUDO_USER
-else
- INSTALL_USER=$USER
-fi
-"""
-base_factory.addStep(Configure(command=WithProperties(maybe_sudo+"""
-$CMD_PREFIX env PATH=/usr/bin:/bin:/usr/sbin:/sbin ./configure --enable-readline \
</del><ins>+base_factory.addStep(SVN(repourl='https://svn.macports.org/repository/macports/trunk/base',
+#base_factory.addStep(SVN(repourl=Interpolate('https://svn.macports.org/repository/macports/%(src::branch:-trunk)s/base'),
+ method="copy"))
+base_factory.addStep(Configure(command=WithProperties("""
+env PATH=/usr/bin:/bin:/usr/sbin:/sbin ./configure --enable-readline \
</ins><span class="cx"> --prefix=%(workdir)s/opt/local \
</span><span class="cx"> --with-applications-dir=%(workdir)s/opt/local/Applications \
</span><del>- --with-install-user=$INSTALL_USER --with-install-group=`id -gn $INSTALL_USER`
-""")))
-base_factory.addStep(Compile(command=maybe_sudo+"""
-$CMD_PREFIX make -j`sysctl -n hw.activecpu`
-"""))
-base_factory.addStep(ShellCommand(command=maybe_sudo+"""
-$CMD_PREFIX make install
-""", name="install", description="install"))
-base_factory.addStep(ShellCommand(command=maybe_sudo+"""
-$CMD_PREFIX make test
-""", name="test", description="test"))
</del><ins>+ --with-install-user=`id -un` \
+ --with-install-group=`id -gn` \
+"""),logfiles={"config.log": "config.log"}))
+base_factory.addStep(Compile(command="make -j`sysctl -n hw.activecpu`"))
+base_factory.addStep(ShellCommand(command="make install", name="install", description="install"))
+base_factory.addStep(ShellCommand(command="make test", name="test", description="test"))
</ins><span class="cx"> base_factory.addStep(ShellCommand(command=WithProperties("make distclean; rm -rf %(workdir)s/opt/local"),
</span><span class="cx"> name="clean", description="clean"))
</span><span class="cx">
</span><del>-
</del><span class="cx"> # custom class to make the file list available on the slave...
</span><del>-class ShellCommandWithPortList(ShellCommand):
- name = 'write portlist file'
- description = 'write portlist file'
</del><ins>+class SetPropertyFromCommandWithPortlist(SetPropertyFromCommand):
+ name = 'generate portlist'
+ description = 'generate portlist'
</ins><span class="cx">
</span><span class="cx"> def setBuild(self, build):
</span><del>- ShellCommand.setBuild(self, build)
</del><ins>+ SetPropertyFromCommand.setBuild(self, build)
</ins><span class="cx">
</span><ins>+ portset = set()
</ins><span class="cx"> # support forced build properties
</span><del>- try:
- portlist = self.getProperty('portlist').strip()
- except:
- portlist = ''
</del><ins>+ if self.getProperty('portlist'):
+ for v in self.getProperty('portlist').strip().split():
+ portset.add(v)
</ins><span class="cx">
</span><del>- portset = set()
</del><span class="cx"> # paths should be dports/category/portdir(/...)
</span><span class="cx"> for f in self.build.allFiles():
</span><span class="cx"> comps = f.split('/')
</span><span class="cx"> if len(comps) >= 3 and comps[0] == 'dports' and comps[1] != '_resources':
</span><span class="cx"> portset.add(comps[2])
</span><del>- portlist += ' ' + ' '.join(portset)
- self.setProperty('portlist', portlist)
</del><span class="cx">
</span><ins>+ self.setProperty('portlist', ' '.join(portset))
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> # can't run with PREFIX/SRC_PREFIX inside the workdir in production,
</span><span class="cx"> # because archives must be built with prefix=/opt/local
</span><span class="cx"> if production:
</span><span class="cx"> prefix='/opt/local'
</span><span class="cx"> src_prefix='/opt/mports'
</span><del>- dlhost='packages@packages.macports.org'
</del><ins>+ dlhost='packages@packages-origin.macports.org'
</ins><span class="cx"> dlpath='/var/www/html/packages'
</span><span class="cx"> else:
</span><del>- prefix='%(workdir)s/opt/local'
- src_prefix='%(workdir)s/opt/mports'
</del><ins>+ prefix='/opt/local'
+ src_prefix='/opt/dports'
+ tools_prefix='/opt/mports'
+
+ #prefix='%(workdir)s/opt/local'
+ #src_prefix='%(workdir)s/opt/mports'
+
</ins><span class="cx"> dlpath='./deployed_archives'
</span><span class="cx"> dlhost=''
</span><span class="cx">
</span><span class="cx"> ulpath='archive_staging'
</span><span class="cx"> ulpath_unique=ulpath+'-%(buildername)s'
</span><span class="cx">
</span><ins>+from buildbot.steps.transfer import DirectoryUpload
+from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.trigger import Trigger
</ins><span class="cx">
</span><del>-ports_factory = BuildFactory()
-# get MPAB itself; we'll do the checkout of base and dports via MPAB's script
-ports_factory.addStep(SVN(svnurl='https://svn.macports.org/repository/macports/contrib/mpab',
- mode="update"))
-ports_factory.addStep(ShellCommand(command=["./mpsync.sh"],
- name="sync",
- description="sync",
- env={'PREFIX': WithProperties(prefix),
- 'SRC_PREFIX': WithProperties(src_prefix),
- 'BASE_UPDATE': 'selfupdate'}))
-ports_factory.addStep(ShellCommandWithPortList(command=WithProperties('rm -f portlist; for portname in %(portlist)s; do for subport in `./subports.tcl $portname`; do echo $subport >> portlist; done; done')))
-# run MPAB on the port list
-ports_factory.addStep(Compile(command=["./mpab", "buildports", "portlist"],
- env={'PREFIX': WithProperties(prefix),
- 'SRC_PREFIX': WithProperties(src_prefix)},
- logfiles={"progress": "progress.log"},
- haltOnFailure=False))
</del><ins>+class TriggerWithPortlist(Trigger):
+ def setTriggerTuple(self,t):
+ self.triggertuple = t
</ins><span class="cx">
</span><del>-ports_factory.addStep(ShellCommand(command=["./gather_archives.sh"],
- name="gather archives",
- description="gather distributable archives",
- env={'PREFIX': WithProperties(prefix),
- 'ULPATH': ulpath}))
</del><ins>+ def getSchedulersAndProperties(self):
+ sp = []
+ for scheduler in self.schedulerNames:
+ for port in self.build.getProperty("subportlist").split():
+ sp.append([scheduler,{"portlist": port}])
+ return sp
+
+
+# -- Port Watcher --
+
+def make_portwatcher_factory(triggerable):
+ portwatcher_factory = BuildFactory()
+ # get mp-buildbot; we'll do the checkout of base and dports via these scripts
+ portbuilder_factory.addStep(SVN(repourl=mpbbsvnurl,alwaysUseLatest=True,preferLastChangedRev=True,mode="incremental",workdir="build/mpbb",haltOnFailure=True))
+
+ portwatcher_factory.addStep(SetPropertyFromCommandWithPortlist(command=WithProperties('for portname in %(portlist)s; do for subport in `./mpbb/tools/subports.tcl $portname`; do echo $subport; done; done' ),
+ property="subportlist"))
+
+ portwatcher_factory.addStep(TriggerWithPortlist(schedulerNames=[triggerable],
+ waitForFinish=True,
+ updateSourceStamp=True))
+
+ # make a logfile summarising the success/failure status for each port
+ #portwatcher_factory.addStep(ShellCommand(command=["./do_status.sh"],
+ # name="status",
+ # description="status",
+ # env={'PREFIX': WithProperties(prefix)
+ # },
+ # logfiles={"portstatus": "portstatus.log"}))
+
+ return portwatcher_factory
+
+from buildbot.process.buildstep import BuildStep
+
+class InfoStep(BuildStep):
+ def __init__(self, title='', **kwargs):
+ BuildStep.__init__(self,**kwargs)
+ self.name = title
+ self.description = title
+ self.descriptionDone = title
+ self.doStepIf = False
+
+# -- Port Builder --
+
+portbuilder_factory = BuildFactory()
+
+# XXX: use InfoStep instead of running a dummy command
+portbuilder_factory.addStep(ShellCommand(command=["/usr/bin/true"],descriptionDone=WithProperties("Port %(portlist)s")))
+#portbuilder_factory.addStep(InfoStep(title=WithProperties("Port %(portlist)s")))
+
+portbuilder_factory.addStep(SVN(repourl=mpbbsvnurl,alwaysUseLatest=True,preferLastChangedRev=True,mode="incremental",workdir="build/mpbb",haltOnFailure=True))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'selfupdate', '--prefix', WithProperties(prefix)],haltOnFailure=True,
+ name="selfupdate", description="selfupdate"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'checkout', '--prefix', WithProperties(prefix), '--svn-url', 'http://linux.mpm16/svn-test/trunk'],
+ timeout=3600,haltOnFailure=True,
+ name="checkout", description="checkout"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'install-dependencies', '--prefix', WithProperties(prefix), '--port', WithProperties('%(portlist)s')],haltOnFailure=True,
+ name="install-dependencies", description="install-dependencies"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'install-port', '--prefix', WithProperties(prefix), '--port', WithProperties('%(portlist)s')],haltOnFailure=True,
+ name="install-port", description="install-port"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'gather-archives', '--prefix', WithProperties(prefix), '--port', WithProperties('%(portlist)s'), '--archive-site', 'http://linux.mpm16/packages', '--staging-dir', ulpath],haltOnFailure=True,
+ name="gather-archives", description="gather-archives"))
+
</ins><span class="cx"> # upload archives from build slave to master
</span><del>-from buildbot.steps.transfer import DirectoryUpload
-ports_factory.addStep(DirectoryUpload(slavesrc=ulpath, masterdest=WithProperties(ulpath_unique)))
</del><ins>+portbuilder_factory.addStep(DirectoryUpload(slavesrc=ulpath, masterdest=WithProperties(ulpath_unique)))
+
+# XXX: move deploy_archives.sh functionality to mp-buildbot
</ins><span class="cx"> # sign generated binaries and sync to download server (if distributable)
</span><del>-from buildbot.steps.master import MasterShellCommand
-ports_factory.addStep(MasterShellCommand(command=["./deploy_archives.sh", WithProperties(ulpath_unique)],
- name="deploy archives",
- description="deploy archives",
- env={'PRIVKEY': privkey,
- 'DLHOST': dlhost,
- 'DLPATH': dlpath}))
</del><ins>+#if production:
+ #portbuilder_factory.addStep(MasterShellCommand(command=["./deploy_archives.sh", WithProperties(ulpath_unique)],
+ # name="deploy archives",
+ # description="deploy archives",
+ # env={'PRIVKEY': privkey,
+ # 'DLHOST': dlhost,
+ # 'DLPATH': dlpath}))
</ins><span class="cx">
</span><del>-# make a logfile summarising the success/failure status for each port
-ports_factory.addStep(ShellCommand(command=["./do_status.sh"],
- name="status",
- description="status",
- env={'PREFIX': WithProperties(prefix)
- },
- logfiles={"portstatus": "portstatus.log"}))
</del><span class="cx"> # TODO: do we want to upload the individual logs so maintainers can review them?
</span><del>-ports_factory.addStep(ShellCommand(command="./cleanup_old.sh",
- name="cleanup",
- description="cleanup",
- env={'PREFIX': WithProperties(prefix),
- 'ULPATH': ulpath}))
</del><ins>+#ports_factory.addStep(ShellCommand(command="./cleanup_old.sh",
+# name="cleanup",
+# description="cleanup",
+# env={'PREFIX': WithProperties(prefix),
+# 'ULPATH': ulpath}))
</ins><span class="cx">
</span><ins>+# === BUILDER CONFIGURATION ===
+
</ins><span class="cx"> from buildbot.config import BuilderConfig
</span><span class="cx">
</span><ins>+# XXX: slavenames assignment should be automatic and more generic
</ins><span class="cx"> portsslaves = {}
</span><span class="cx"> baseslaves = {}
</span><span class="cx"> if production:
</span><span class="lines">@@ -270,20 +324,27 @@
</span><span class="cx"> for plat in build_platforms:
</span><span class="cx"> baseslaves[plat] = filter(lambda x: x.endswith(plat+"-base"), slavenames)
</span><span class="cx"> portsslaves[plat] = filter(lambda x: x.endswith(plat+"-ports"), slavenames)
</span><del>-else:
- slavenames = ["snowleopard-x86_64"]
- portsslaves = {build_platforms[0]:slavenames[0]}
- baseslaves = {build_platforms[0]:slavenames[0]}
</del><ins>+#else:
+ #slavenames = ["snowleopard-x86_64"]
+ #slavenames = ["base-10.6-x86_64","10.6-x86_64","10.9-x86_64"]
+ #portsslaves = {build_platforms[0]:slavenames[0],build_platforms[1]:slavenames[1]}
+ #baseslaves = {build_platforms[0]:slavenames[0],build_platforms[1]:slavenames[1]}
</ins><span class="cx">
</span><span class="cx"> c['builders']=[]
</span><span class="cx"> for plat in build_platforms:
</span><del>- c['builders']+= [BuilderConfig(name="buildbase-"+plat,
- slavenames=baseslaves[plat],
- factory=base_factory)]
- if 'linux' not in plat:
- c['builders']+= [BuilderConfig(name="buildports-"+plat,
- slavenames=portsslaves[plat],
- factory=ports_factory)]
</del><ins>+ c['builders']+= [BuilderConfig(name="base-"+plat,
+ slavenames=["base-"+plat],
+ factory=base_factory,
+ env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin"})]
+ if 'linux' not in plat:
+ c['builders']+= [BuilderConfig(name="portwatcher-"+plat,
+ slavenames=[plat],
+ factory=make_portwatcher_factory("triggerable-portbuilder-"+plat),
+ env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin"})]
+ c['builders']+= [BuilderConfig(name="portbuilder-"+plat,
+ slavenames=[plat],
+ factory=portbuilder_factory,
+ env={"PATH": tools_prefix+"/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin"})]
</ins><span class="cx">
</span><span class="cx"> ####### STATUS TARGETS
</span><span class="cx">
</span><span class="lines">@@ -295,30 +356,24 @@
</span><span class="cx">
</span><span class="cx"> from buildbot.status import html
</span><span class="cx"> from buildbot.status.web import auth, authz
</span><del>-if production:
- from buildbot.status.web.auth import HTPasswdAuth
- htauth = HTPasswdAuth('htpasswd')
- authz_cfg=authz.Authz(
- auth=htauth,
- gracefulShutdown = 'auth',
- forceBuild = 'auth',
- forceAllBuilds = 'auth',
- pingBuilder = 'auth',
- stopBuild = 'auth',
- stopAllBuilds = 'auth',
- cancelPendingBuild = 'auth',
- )
-else:
- authz_cfg=authz.Authz(
- gracefulShutdown = True,
- forceBuild = True,
- forceAllBuilds = True,
- pingBuilder = True,
- stopBuild = True,
- stopAllBuilds = True,
- cancelPendingBuild = True
- )
</del><span class="cx">
</span><ins>+from buildbot.status.web.auth import HTPasswdAuth
+
+# add new users with crypt, not md5/sha1:
+# htpasswd -d -n $USER
+htauth = HTPasswdAuth(htpasswdfile)
+
+authz_cfg=authz.Authz(
+ auth=htauth,
+ gracefulShutdown = 'auth',
+ forceBuild = 'auth',
+ forceAllBuilds = 'auth',
+ pingBuilder = 'auth',
+ stopBuild = 'auth',
+ stopAllBuilds = 'auth',
+ cancelPendingBuild = 'auth',
+)
+
</ins><span class="cx"> if production:
</span><span class="cx"> # send mail about base failures to users on the blamelist
</span><span class="cx"> from buildbot.status.mail import MailNotifier
</span><span class="lines">@@ -337,18 +392,18 @@
</span><span class="cx"> failedPorts = set()
</span><span class="cx"> interestedUsers = set()
</span><span class="cx">
</span><ins>+ # XXX: needs to be rewritten for the new steps of mp-buildbot
</ins><span class="cx"> statusStep = [x for x in build.getSteps() if x.getName() == "status"][0]
</span><span class="cx"> statusLog = [x for x in statusStep.getLogs() if x.getName() == "portstatus"][0]
</span><span class="cx"> for line in statusLog.getText().splitlines():
</span><span class="cx"> halves = line.split()
</span><span class="cx"> if halves[0] == "[FAIL]":
</span><span class="cx"> failedPorts.add(halves[1])
</span><del>-
- fakeAddresses = {'nomaintainer', 'nomaintainer@macports.org', 'openmaintainer', 'openmaintainer@macports.org'}
</del><ins>+
</ins><span class="cx"> for p in failedPorts:
</span><span class="cx"> output = subprocess.Popen(['/opt/local/bin/port', 'info', '--index', '--maintainers', '--line', p], stdout=subprocess.PIPE).communicate()[0].strip()
</span><span class="cx"> for m in output.split(','):
</span><del>- if m not in fakeAddresses:
</del><ins>+ if m != "nomaintainer@macports.org" and m != "openmaintainer@macports.org":
</ins><span class="cx"> interestedUsers.add(m)
</span><span class="cx">
</span><span class="cx"> ss = build.getSourceStamp()
</span><span class="lines">@@ -382,20 +437,24 @@
</span><span class="cx"> c['title'] = "MacPorts"
</span><span class="cx"> c['titleURL'] = "http://www.macports.org/"
</span><span class="cx">
</span><del>-if production:
- c['buildbotURL'] = "http://build.macports.org/"
- c['status'].append(html.WebStatus(http_port=8710, authz=authz_cfg))
-else:
- c['buildbotURL'] = "http://localhost:8010/"
- c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
</del><ins>+c['buildbotURL'] = buildboturl
+c['status'].append(html.WebStatus(http_port=httpport, authz=authz_cfg))
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> ####### DB URL
</span><span class="cx">
</span><span class="cx"> # This specifies what database buildbot uses to store change and scheduler
</span><span class="cx"> # state. You can leave this at its default for all but the largest
</span><span class="cx"> # installations.
</span><del>-c['db_url'] = "sqlite:///state.sqlite"
</del><ins>+#c['db_url'] = "sqlite:///state.sqlite"
</ins><span class="cx">
</span><ins>+c['db'] = {
+ # This specifies what database buildbot uses to store its state. You can leave
+ # this at its default for all but the largest installations.
+ 'db_url' : "sqlite:///state.sqlite",
+}
+
+
</ins><span class="cx"> ######## Cache settings
</span><span class="cx"> c['buildHorizon'] = 10000
</span><span class="cx"> c['logHorizon'] = 5000
</span></span></pre></div>
<a id="contribbuildbottestslavesjsonsample"></a>
<div class="addfile"><h4>Added: contrib/buildbot-test/slaves.json.sample (0 => 146776)</h4>
<pre class="diff"><span>
<span class="info">--- contrib/buildbot-test/slaves.json.sample (rev 0)
+++ contrib/buildbot-test/slaves.json.sample 2016-03-17 09:43:06 UTC (rev 146776)
</span><span class="lines">@@ -0,0 +1,4 @@
</span><ins>+{
+ "base-10.6-x86_64": "***",
+ "10.6-x86_64": "***",
+}
</ins></span></pre>
</div>
</div>
</body>
</html>