<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14640] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14640">14640</a></dd>
<dt>Author</dt> <dd>dre@apple.com</dd>
<dt>Date</dt> <dd>2015-03-31 13:20:48 -0700 (Tue, 31 Mar 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Add support for spawning and connecting to memcached over unix domain sockets, which is now the default. Set Memcached.Pools.Default.MemcacheSocket to '' to use TCP. Also two new tests, one for each mode, to verify we can spawn / connect to / interact with / despawn memcached. In unix socket mode, validate safe file permissions on the socket file.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkcalendarservertapcaldavpy">CalendarServer/trunk/calendarserver/tap/caldav.py</a></li>
<li><a href="#CalendarServertrunkcalendarservertaptesttest_caldavpy">CalendarServer/trunk/calendarserver/tap/test/test_caldav.py</a></li>
<li><a href="#CalendarServertrunktwistedcaldavmemcachepoolpy">CalendarServer/trunk/twistedcaldav/memcachepool.py</a></li>
<li><a href="#CalendarServertrunktwistedcaldavstdconfigpy">CalendarServer/trunk/twistedcaldav/stdconfig.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkcalendarservertapcaldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tap/caldav.py (14639 => 14640)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tap/caldav.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2015-03-31 20:20:48 UTC (rev 14640)
</span><span class="lines">@@ -1214,7 +1214,47 @@
</span><span class="cx"> config.BindAddresses = [""]
</span><span class="cx"> return config.BindAddresses
</span><span class="cx">
</span><ins>+ def _spawnMemcached(self, monitor=None):
+ """
+ Optionally start memcached through the specified ProcessMonitor,
+ or if monitor is None, use Popen.
+ """
+ for name, pool in config.Memcached.Pools.items():
+ if pool.ServerEnabled:
+ memcachedArgv = [
+ config.Memcached.memcached,
+ "-U", "0",
+ ]
+ # Use Unix domain sockets by default
+ if pool.MemcacheSocket is not '':
+ memcachedArgv.extend([
+ "-s", str(pool.MemcacheSocket),
+ ])
+ else:
+ # ... or INET sockets
+ memcachedArgv.extend([
+ "-p", str(pool.Port),
+ "-l", pool.BindAddress,
+ ])
+ if config.Memcached.MaxMemory is not 0:
+ memcachedArgv.extend(
+ ["-m", str(config.Memcached.MaxMemory)]
+ )
+ if config.UserName:
+ memcachedArgv.extend(["-u", config.UserName])
+ memcachedArgv.extend(config.Memcached.Options)
+ print(
+ "Adding memcached service for pool:", name, memcachedArgv
+ )
+ if monitor is not None:
+ monitor.addProcess(
+ "memcached-{}".format(name), memcachedArgv,
+ env=PARENT_ENVIRONMENT
+ )
+ else:
+ Popen(memcachedArgv)
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def makeService_Single(self, options):
</span><span class="cx"> """
</span><span class="cx"> Create a service to be used in a single-process, stand-alone
</span><span class="lines">@@ -1355,30 +1395,10 @@
</span><span class="cx"> config.AccessLogFile,
</span><span class="cx"> )
</span><span class="cx">
</span><del>- # Optionally launch memcached. Note, this is not going through a
</del><ins>+ # Maybe spawn memcached. Note, this is not going through a
</ins><span class="cx"> # ProcessMonitor because there is code elsewhere that needs to
</span><del>- # access memcached before startService() gets called, so we're just
- # directly using Popen to spawn memcached.
- for name, pool in config.Memcached.Pools.items():
- if pool.ServerEnabled:
- self.log.info(
- "Adding memcached service for pool: {name}",
- name=name, pool=pool
- )
- memcachedArgv = [
- config.Memcached.memcached,
- "-p", str(pool.Port),
- "-l", pool.BindAddress,
- "-U", "0",
- ]
- if config.Memcached.MaxMemory is not 0:
- memcachedArgv.extend(
- ["-m", str(config.Memcached.MaxMemory)]
- )
- if config.UserName:
- memcachedArgv.extend(["-u", config.UserName])
- memcachedArgv.extend(config.Memcached.Options)
- Popen(memcachedArgv)
</del><ins>+ # access memcached before startService() gets called
+ self._spawnMemcached(monitor=None)
</ins><span class="cx">
</span><span class="cx"> return self.storageService(
</span><span class="cx"> slaveSvcCreator, logObserver, uid=uid, gid=gid
</span><span class="lines">@@ -1710,29 +1730,8 @@
</span><span class="cx"> )
</span><span class="cx"> memoryLimiter.setServiceParent(s)
</span><span class="cx">
</span><del>- for name, pool in config.Memcached.Pools.items():
- if pool.ServerEnabled:
- self.log.info(
- "Adding memcached service for pool: {name}",
- name=name, pool=pool
- )
- memcachedArgv = [
- config.Memcached.memcached,
- "-p", str(pool.Port),
- "-l", pool.BindAddress,
- "-U", "0",
- ]
- if config.Memcached.MaxMemory is not 0:
- memcachedArgv.extend(
- ["-m", str(config.Memcached.MaxMemory)]
- )
- if config.UserName:
- memcachedArgv.extend(["-u", config.UserName])
- memcachedArgv.extend(config.Memcached.Options)
- monitor.addProcess(
- "memcached-{}".format(name), memcachedArgv,
- env=PARENT_ENVIRONMENT
- )
</del><ins>+ # Maybe spawn memcached through a ProcessMonitor
+ self._spawnMemcached(monitor=monitor)
</ins><span class="cx">
</span><span class="cx"> # Open the socket(s) to be inherited by the slaves
</span><span class="cx"> inheritFDs = []
</span></span></pre></div>
<a id="CalendarServertrunkcalendarservertaptesttest_caldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tap/test/test_caldav.py (14639 => 14640)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2015-03-31 20:20:48 UTC (rev 14640)
</span><span class="lines">@@ -18,7 +18,9 @@
</span><span class="cx"> import os
</span><span class="cx"> import stat
</span><span class="cx"> import grp
</span><ins>+import random
</ins><span class="cx">
</span><ins>+from time import sleep
</ins><span class="cx"> from os.path import dirname, abspath
</span><span class="cx"> from collections import namedtuple
</span><span class="cx">
</span><span class="lines">@@ -30,6 +32,8 @@
</span><span class="cx"> from twisted.python.usage import Options, UsageError
</span><span class="cx"> from twisted.python.procutils import which
</span><span class="cx">
</span><ins>+from twisted.runner.procmon import ProcessMonitor
+
</ins><span class="cx"> from twisted.internet.interfaces import IProcessTransport, IReactorProcess
</span><span class="cx"> from twisted.internet.protocol import ServerFactory
</span><span class="cx"> from twisted.internet.defer import Deferred, inlineCallbacks, succeed
</span><span class="lines">@@ -46,6 +50,7 @@
</span><span class="cx"> from txweb2.log import LogWrapperResource
</span><span class="cx"> from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
</span><span class="cx">
</span><ins>+from twistedcaldav import memcacheclient
</ins><span class="cx"> from twistedcaldav.config import config, ConfigDict, ConfigurationError
</span><span class="cx"> from twistedcaldav.resource import AuthenticationWrapper
</span><span class="cx"> from twistedcaldav.stdconfig import DEFAULT_CONFIG
</span><span class="lines">@@ -437,7 +442,63 @@
</span><span class="cx"> self.assertEquals(socketService.gid, self.alternateGroup)
</span><span class="cx">
</span><span class="cx">
</span><ins>+class MemcacheSpawner(TestCase):
</ins><span class="cx">
</span><ins>+ def setUp(self):
+ super(MemcacheSpawner, self).setUp()
+ self.monitor = ProcessMonitor()
+ self.monitor.startService()
+ self.socket = os.path.abspath("memcache.sock")
+ self.patch(config.Memcached.Pools.Default, "ServerEnabled", True)
+
+
+ def test_memcacheUnix(self):
+ """
+ Spawn a memcached process listening on a unix socket that becomes
+ connectable in no more than one second. Connect and interact.
+ Verify secure file permissions on the socket file.
+ """
+ self.patch(config.Memcached.Pools.Default, "MemcacheSocket", self.socket)
+ CalDAVServiceMaker()._spawnMemcached(monitor=self.monitor)
+ sleep(1)
+ mc = memcacheclient.Client(["unix:{}".format(self.socket)], debug=1)
+ rando = random.random()
+ mc.set("the_answer", rando)
+ self.assertEquals(rando, mc.get("the_answer"))
+ # The socket file should not be usable to other users
+ st = os.stat(self.socket)
+ self.assertTrue(str(oct(st.st_mode)).endswith("00"))
+ mc.disconnect_all()
+
+
+ def test_memcacheINET(self):
+ """
+ Spawn a memcached process listening on a network socket that becomes
+ connectable in no more than one second. Interact with it.
+ """
+ self.patch(config.Memcached.Pools.Default, "MemcacheSocket", "")
+ ba = config.Memcached.Pools.Default.BindAddress
+ bp = config.Memcached.Pools.Default.Port
+ CalDAVServiceMaker()._spawnMemcached(monitor=self.monitor)
+ sleep(1)
+ mc = memcacheclient.Client(["{}:{}".format(ba, bp)], debug=1)
+ rando = random.random()
+ mc.set("the_password", rando)
+ self.assertEquals(rando, mc.get("the_password"))
+ mc.disconnect_all()
+
+
+ def tearDown(self):
+ """
+ Verify that our spawned memcached can be reaped in no more than
+ one second - if not we'll get reactor unclean failures.
+ """
+ self.monitor.stopService()
+ sleep(1)
+ if os.path.exists(self.socket):
+ os.remove(self.socket)
+
+
</ins><span class="cx"> class ProcessMonitorTests(CalDAVServiceMakerTestBase):
</span><span class="cx">
</span><span class="cx"> def configure(self):
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavmemcachepoolpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/memcachepool.py (14639 => 14640)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/memcachepool.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/twistedcaldav/memcachepool.py 2015-03-31 20:20:48 UTC (rev 14640)
</span><span class="lines">@@ -23,6 +23,7 @@
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twext.internet.gaiendpoint import GAIEndpoint
</span><span class="cx"> from twext.internet.adaptendpoint import connect
</span><ins>+from twisted.internet.endpoints import UNIXClientEndpoint
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -446,10 +447,15 @@
</span><span class="cx"> from twisted.internet import reactor
</span><span class="cx"> for name, pool in pools.items():
</span><span class="cx"> if pool["ClientEnabled"]:
</span><ins>+ if pool["MemcacheSocket"] is not '':
+ ep = UNIXClientEndpoint(reactor, pool["MemcacheSocket"])
+ else:
+ ep = GAIEndpoint(reactor, pool["BindAddress"], pool["Port"])
+
</ins><span class="cx"> _installPool(
</span><span class="cx"> name,
</span><span class="cx"> pool["HandleCacheTypes"],
</span><del>- GAIEndpoint(reactor, pool["BindAddress"], pool["Port"]),
</del><ins>+ ep,
</ins><span class="cx"> maxClients,
</span><span class="cx"> reactor,
</span><span class="cx"> )
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavstdconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py (14639 => 14640)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-03-31 20:20:48 UTC (rev 14640)
</span><span class="lines">@@ -932,6 +932,9 @@
</span><span class="cx"> "MaxClients": 5,
</span><span class="cx"> "Pools": {
</span><span class="cx"> "Default": {
</span><ins>+ # A unix socket used for communication with memcached.
+ # If blank, then an AF_INET socket is used instead.
+ "MemcacheSocket": "memcache.sock",
</ins><span class="cx"> "ClientEnabled": True,
</span><span class="cx"> "ServerEnabled": True,
</span><span class="cx"> "BindAddress": "127.0.0.1",
</span><span class="lines">@@ -1200,6 +1203,7 @@
</span><span class="cx"> ("RunRoot", "PIDFile"),
</span><span class="cx"> ("RunRoot", ("Stats", "UnixStatsSocket",)),
</span><span class="cx"> ("RunRoot", "ControlSocket"),
</span><ins>+ ("RunRoot", ("Memcached", "Pools", "Default", "MemcacheSocket")),
</ins><span class="cx"> ("RunRoot", ("DirectoryProxy", "SocketPath",)),
</span><span class="cx"> ]
</span><span class="cx">
</span></span></pre>
</div>
</div>
</body>
</html>