<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[152630] contrib/buildbot-test</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="https://trac.macports.org/changeset/152630">152630</a></dd>
<dt>Author</dt> <dd>raimue@macports.org</dd>
<dt>Date</dt> <dd>2016-09-13 16:52:14 -0700 (Tue, 13 Sep 2016)</dd>
</dl>
<h3>Log Message</h3>
<pre>buildbot: add deploy config for www/guide
This adds preliminary support for deploying generated static files with rsync
to the host running the web server. The builders and schedulers will only be
added if the deploy configuration is filled.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#contribbuildbottestconfigjsonsample">contrib/buildbot-test/config.json.sample</a></li>
<li><a href="#contribbuildbottestmastercfg">contrib/buildbot-test/master.cfg</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="contribbuildbottestconfigjsonsample"></a>
<div class="modfile"><h4>Modified: contrib/buildbot-test/config.json.sample (152629 => 152630)</h4>
<pre class="diff"><span>
<span class="info">--- contrib/buildbot-test/config.json.sample 2016-09-13 20:48:39 UTC (rev 152629)
+++ contrib/buildbot-test/config.json.sample 2016-09-13 23:52:14 UTC (rev 152630)
</span><span class="lines">@@ -9,5 +9,21 @@
</span><span class="cx"> "svnurl": "https://svn.macports.org/repository/macports/trunk",
</span><span class="cx"> "archivesite": "https://packages.macports.org",
</span><span class="cx"> "slaveprefix": "/opt/local",
</span><del>- "toolsprefix": "/opt/mports"
</del><ins>+ "toolsprefix": "/opt/mports",
+ "deploy": {
+ "www": {
+ "host": "",
+ "user": "",
+ "sshkeyfile": "",
+ "sshknownhostsfile": "ssh_known_hosts",
+ "destpath": ""
+ },
+ "guide": {
+ "host": "",
+ "user": "",
+ "sshkeyfile": "",
+ "sshknownhostsfile": "ssh_known_hosts",
+ "destpath": ""
+ }
+ }
</ins><span class="cx"> }
</span></span></pre></div>
<a id="contribbuildbottestmastercfg"></a>
<div class="modfile"><h4>Modified: contrib/buildbot-test/master.cfg (152629 => 152630)</h4>
<pre class="diff"><span>
<span class="info">--- contrib/buildbot-test/master.cfg 2016-09-13 20:48:39 UTC (rev 152629)
+++ contrib/buildbot-test/master.cfg 2016-09-13 23:52:14 UTC (rev 152630)
</span><span class="lines">@@ -37,7 +37,8 @@
</span><span class="cx"> 'svnurl': "https://svn.macports.org/repository/macports/trunk",
</span><span class="cx"> 'archivesite': "https://packages.macports.org",
</span><span class="cx"> 'slaveprefix': "/opt/local",
</span><del>- 'toolsprefix': "/opt/mports"
</del><ins>+ 'toolsprefix': "/opt/mports",
+ 'deploy': {}
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> if os.path.exists(_path('config.json')):
</span><span class="lines">@@ -146,7 +147,6 @@
</span><span class="cx"> portwatcher_buildernames = ["ports-"+plat+"-watcher" for plat in build_platforms if 'linux' not in plat and '10.5_ppc' != plat]
</span><span class="cx"> portbuilder_buildernames = ["ports-"+plat+"-builder" for plat in build_platforms if 'linux' not in plat and '10.5_ppc' != plat]
</span><span class="cx"> portbuilder_triggerables = ["ports-"+plat+"-trigger" for plat in build_platforms if 'linux' not in plat and '10.5_ppc' != plat]
</span><del>-docbuilder_buildernames = ["docs-www", "docs-guide"] if "docs" in slavedata['slaves'].keys() else []
</del><span class="cx">
</span><span class="cx"> from buildbot.schedulers.basic import SingleBranchScheduler
</span><span class="cx"> from buildbot.schedulers.forcesched import ForceScheduler
</span><span class="lines">@@ -185,25 +185,32 @@
</span><span class="cx"> default="",
</span><span class="cx"> size=30,
</span><span class="cx"> required=True)
</span><del>- ]),
- SingleBranchScheduler(
- name="www",
- treeStableTimer=300,
- change_filter = wwwfilter,
- builderNames=docbuilder_buildernames),
- SingleBranchScheduler(
- name="guide",
- treeStableTimer=300,
- change_filter = guidefilter,
- builderNames=docbuilder_buildernames),
- ForceScheduler(
- name="www_force",
- builderNames=docbuilder_buildernames),
- ForceScheduler(
- name="guide_force",
- builderNames=docbuilder_buildernames),
</del><ins>+ ])
</ins><span class="cx"> ]
</span><span class="cx">
</span><ins>+
+if 'www' in config['deploy']:
+ c['schedulers'] += [
+ SingleBranchScheduler(
+ name="www",
+ treeStableTimer=300,
+ change_filter = wwwfilter,
+ builderNames=["docs-www"]),
+ ForceScheduler(
+ name="www_force",
+ builderNames=["docs-www"])]
+
+if 'guide' in config['deploy']:
+ c['schedulers'] += [
+ SingleBranchScheduler(
+ name="guide",
+ treeStableTimer=300,
+ change_filter = guidefilter,
+ builderNames=["docs-guide"]),
+ ForceScheduler(
+ name="guide_force",
+ builderNames=["docs-www"])]
+
</ins><span class="cx"> for i in range(len(portbuilder_buildernames)):
</span><span class="cx"> c['schedulers'].append(Triggerable(
</span><span class="cx"> name=portbuilder_triggerables[i],
</span><span class="lines">@@ -297,7 +304,7 @@
</span><span class="cx"> ulpath='archive_staging'
</span><span class="cx"> ulpath_unique=ulpath+'-%(buildername)s'
</span><span class="cx">
</span><del>-from buildbot.steps.transfer import DirectoryUpload
</del><ins>+from buildbot.steps.transfer import FileDownload, DirectoryUpload
</ins><span class="cx"> from buildbot.steps.master import MasterShellCommand
</span><span class="cx"> from buildbot.steps.trigger import Trigger
</span><span class="cx">
</span><span class="lines">@@ -460,25 +467,73 @@
</span><span class="cx"> alwaysRun=True))
</span><span class="cx">
</span><span class="cx">
</span><del>-docs_www_factory = BuildFactory()
-docs_www_factory.addStep(SVN(
- repourl='https://svn.macports.org/repository/macports/trunk/www',
- method="copy"))
-# TODO: validate/lint files
-# TODO: deploy generated files
-# TODO: incremental mode with cleanup?
</del><ins>+def make_rsync_deploy_steps(host, user, sshkeyfile, sshknownhostsfile, srcpath, destpath):
+ return [
+ FileDownload(name="ssh key",
+ description="transferring",
+ descriptionDone="transfer",
+ mastersrc=sshkeyfile,
+ slavedest="ssh_key",
+ mode=0600),
+ FileDownload(name="ssh known_hosts",
+ description="transferring",
+ descriptionDone="transfer",
+ mastersrc=sshknownhostsfile,
+ slavedest="ssh_known_hosts",
+ mode=0600),
+ ShellCommand(name="rsync",
+ description="deploying",
+ descriptionDone="deploy",
+ command="rsync -avzhC --delay-updates --delete-delay %s/ %s@%s:%s/" % (srcpath, user, host, destpath),
+ env={'RSYNC_RSH': "ssh -i ssh_key -oUserKnownHostsFile=ssh_known_hosts"})
+ ]
</ins><span class="cx">
</span><del>-docs_guide_factory = BuildFactory()
-docs_guide_factory.addStep(SVN(
- repourl='https://svn.macports.org/repository/macports/trunk/doc-new',
- method="copy"))
-# TODO: check for existance of tools in toolsprefix?
-docs_guide_factory.addStep(Compile(command="make validate"))
-docs_guide_factory.addStep(Compile(command="make all"))
-# TODO: deploy generated files
-# TODO: incremental mode with cleanup?
</del><ins>+if 'www' in config['deploy']:
+ docs_www_factory = BuildFactory()
+ # TODO: incremental mode with cleanup?
+ docs_www_factory.addStep(SVN(
+ repourl='https://svn.macports.org/repository/macports/trunk/www',
+ mode="full",
+ method="copy",
+ workdir="www"))
+ # TODO: validate/lint files
+ docs_www_factory.addSteps(
+ make_rsync_deploy_steps(
+ host=config['deploy']['www']['host'],
+ user=config['deploy']['www']['user'],
+ sshkeyfile=config['deploy']['www']['sshkeyfile'],
+ sshknownhostsfile=config['deploy']['www']['sshknownhostsfile'],
+ srcpath="www",
+ destpath=config['deploy']['www']['destpath']))
</ins><span class="cx">
</span><ins>+if 'guide' in config['deploy']:
+ docs_guide_factory = BuildFactory()
+ # TODO: incremental mode with cleanup?
+ docs_guide_factory.addStep(SVN(
+ repourl='https://svn.macports.org/repository/macports/trunk/doc-new',
+ mode="full",
+ method="copy",
+ workdir="guide"))
+ # TODO: check for existence of tools in toolsprefix
+ docs_guide_factory.addStep(Compile(
+ name="validate",
+ description="validating",
+ descriptionDone="validate",
+ command="make validate",
+ workdir="guide"))
+ docs_guide_factory.addStep(Compile(
+ command="make all",
+ workdir="guide"))
+ docs_guide_factory.addSteps(
+ make_rsync_deploy_steps(
+ host=config['deploy']['guide']['host'],
+ user=config['deploy']['guide']['user'],
+ sshkeyfile=config['deploy']['guide']['sshkeyfile'],
+ sshknownhostsfile=config['deploy']['guide']['sshknownhostsfile'],
+ srcpath="guide",
+ destpath=config['deploy']['guide']['destpath']))
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> # === BUILDER CONFIGURATION ===
</span><span class="cx">
</span><span class="cx"> from buildbot.config import BuilderConfig
</span><span class="lines">@@ -525,20 +580,22 @@
</span><span class="cx"> tags=["portbuilder", os_version],
</span><span class="cx"> env=merge_dicts(env_buildinfo, {"PATH": path_ports}))]
</span><span class="cx">
</span><del>-if "docs" in slavedata['slaves'].keys():
- c['builders'] += [
</del><ins>+if 'www' in config['deploy']:
+ c['builders'].append(
</ins><span class="cx"> BuilderConfig(
</span><span class="cx"> name="docs-www",
</span><span class="cx"> slavenames=["docs"],
</span><span class="cx"> factory=docs_www_factory,
</span><span class="cx"> tags=["docs", "www"],
</span><del>- env=merge_dicts(env_buildinfo, {"PATH": path_ports})),
</del><ins>+ env=merge_dicts(env_buildinfo, {"PATH": path_ports})))
+if 'guide' in config['deploy']:
+ c['builders'].append(
</ins><span class="cx"> BuilderConfig(
</span><span class="cx"> name="docs-guide",
</span><span class="cx"> slavenames=["docs"],
</span><span class="cx"> factory=docs_guide_factory,
</span><span class="cx"> tags=["docs", "guide"],
</span><del>- env=merge_dicts(env_buildinfo, {"PATH": path_ports}))]
</del><ins>+ env=merge_dicts(env_buildinfo, {"PATH": path_ports})))
</ins><span class="cx">
</span><span class="cx"> ####### STATUS TARGETS
</span><span class="cx">
</span></span></pre>
</div>
</div>
</body>
</html>