[CalendarServer-changes] [14640] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Mar 31 13:20:49 PDT 2015
Revision: 14640
http://trac.calendarserver.org//changeset/14640
Author: dre at apple.com
Date: 2015-03-31 13:20:48 -0700 (Tue, 31 Mar 2015)
Log Message:
-----------
Add support for spawning and connecting to memcached over unix domain sockets, which is now the default. Set Memcached.Pools.Default.MemcacheSocket to '' to use TCP. Also two new tests, one for each mode, to verify we can spawn / connect to / interact with / despawn memcached. In unix socket mode, validate safe file permissions on the socket file.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
CalendarServer/trunk/twistedcaldav/memcachepool.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2015-03-31 20:20:48 UTC (rev 14640)
@@ -1214,7 +1214,47 @@
config.BindAddresses = [""]
return config.BindAddresses
+ def _spawnMemcached(self, monitor=None):
+ """
+ Optionally start memcached through the specified ProcessMonitor,
+ or if monitor is None, use Popen.
+ """
+ for name, pool in config.Memcached.Pools.items():
+ if pool.ServerEnabled:
+ memcachedArgv = [
+ config.Memcached.memcached,
+ "-U", "0",
+ ]
+ # Use Unix domain sockets by default
+ if pool.MemcacheSocket is not '':
+ memcachedArgv.extend([
+ "-s", str(pool.MemcacheSocket),
+ ])
+ else:
+ # ... or INET sockets
+ memcachedArgv.extend([
+ "-p", str(pool.Port),
+ "-l", pool.BindAddress,
+ ])
+ if config.Memcached.MaxMemory is not 0:
+ memcachedArgv.extend(
+ ["-m", str(config.Memcached.MaxMemory)]
+ )
+ if config.UserName:
+ memcachedArgv.extend(["-u", config.UserName])
+ memcachedArgv.extend(config.Memcached.Options)
+ print(
+ "Adding memcached service for pool:", name, memcachedArgv
+ )
+ if monitor is not None:
+ monitor.addProcess(
+ "memcached-{}".format(name), memcachedArgv,
+ env=PARENT_ENVIRONMENT
+ )
+ else:
+ Popen(memcachedArgv)
+
def makeService_Single(self, options):
"""
Create a service to be used in a single-process, stand-alone
@@ -1355,30 +1395,10 @@
config.AccessLogFile,
)
- # Optionally launch memcached. Note, this is not going through a
+ # Maybe spawn memcached. Note, this is not going through a
# ProcessMonitor because there is code elsewhere that needs to
- # access memcached before startService() gets called, so we're just
- # directly using Popen to spawn memcached.
- for name, pool in config.Memcached.Pools.items():
- if pool.ServerEnabled:
- self.log.info(
- "Adding memcached service for pool: {name}",
- name=name, pool=pool
- )
- memcachedArgv = [
- config.Memcached.memcached,
- "-p", str(pool.Port),
- "-l", pool.BindAddress,
- "-U", "0",
- ]
- if config.Memcached.MaxMemory is not 0:
- memcachedArgv.extend(
- ["-m", str(config.Memcached.MaxMemory)]
- )
- if config.UserName:
- memcachedArgv.extend(["-u", config.UserName])
- memcachedArgv.extend(config.Memcached.Options)
- Popen(memcachedArgv)
+ # access memcached before startService() gets called
+ self._spawnMemcached(monitor=None)
return self.storageService(
slaveSvcCreator, logObserver, uid=uid, gid=gid
@@ -1710,29 +1730,8 @@
)
memoryLimiter.setServiceParent(s)
- for name, pool in config.Memcached.Pools.items():
- if pool.ServerEnabled:
- self.log.info(
- "Adding memcached service for pool: {name}",
- name=name, pool=pool
- )
- memcachedArgv = [
- config.Memcached.memcached,
- "-p", str(pool.Port),
- "-l", pool.BindAddress,
- "-U", "0",
- ]
- if config.Memcached.MaxMemory is not 0:
- memcachedArgv.extend(
- ["-m", str(config.Memcached.MaxMemory)]
- )
- if config.UserName:
- memcachedArgv.extend(["-u", config.UserName])
- memcachedArgv.extend(config.Memcached.Options)
- monitor.addProcess(
- "memcached-{}".format(name), memcachedArgv,
- env=PARENT_ENVIRONMENT
- )
+ # Maybe spawn memcached through a ProcessMonitor
+ self._spawnMemcached(monitor=monitor)
# Open the socket(s) to be inherited by the slaves
inheritFDs = []
Modified: CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2015-03-31 20:20:48 UTC (rev 14640)
@@ -18,7 +18,9 @@
import os
import stat
import grp
+import random
+from time import sleep
from os.path import dirname, abspath
from collections import namedtuple
@@ -30,6 +32,8 @@
from twisted.python.usage import Options, UsageError
from twisted.python.procutils import which
+from twisted.runner.procmon import ProcessMonitor
+
from twisted.internet.interfaces import IProcessTransport, IReactorProcess
from twisted.internet.protocol import ServerFactory
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
@@ -46,6 +50,7 @@
from txweb2.log import LogWrapperResource
from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
+from twistedcaldav import memcacheclient
from twistedcaldav.config import config, ConfigDict, ConfigurationError
from twistedcaldav.resource import AuthenticationWrapper
from twistedcaldav.stdconfig import DEFAULT_CONFIG
@@ -437,7 +442,63 @@
self.assertEquals(socketService.gid, self.alternateGroup)
+class MemcacheSpawner(TestCase):
+ def setUp(self):
+ super(MemcacheSpawner, self).setUp()
+ self.monitor = ProcessMonitor()
+ self.monitor.startService()
+ self.socket = os.path.abspath("memcache.sock")
+ self.patch(config.Memcached.Pools.Default, "ServerEnabled", True)
+
+
+ def test_memcacheUnix(self):
+ """
+ Spawn a memcached process listening on a unix socket that becomes
+ connectable in no more than one second. Connect and interact.
+ Verify secure file permissions on the socket file.
+ """
+ self.patch(config.Memcached.Pools.Default, "MemcacheSocket", self.socket)
+ CalDAVServiceMaker()._spawnMemcached(monitor=self.monitor)
+ sleep(1)
+ mc = memcacheclient.Client(["unix:{}".format(self.socket)], debug=1)
+ rando = random.random()
+ mc.set("the_answer", rando)
+ self.assertEquals(rando, mc.get("the_answer"))
+ # The socket file should not be usable to other users
+ st = os.stat(self.socket)
+ self.assertTrue(str(oct(st.st_mode)).endswith("00"))
+ mc.disconnect_all()
+
+
+ def test_memcacheINET(self):
+ """
+ Spawn a memcached process listening on a network socket that becomes
+ connectable in no more than one second. Interact with it.
+ """
+ self.patch(config.Memcached.Pools.Default, "MemcacheSocket", "")
+ ba = config.Memcached.Pools.Default.BindAddress
+ bp = config.Memcached.Pools.Default.Port
+ CalDAVServiceMaker()._spawnMemcached(monitor=self.monitor)
+ sleep(1)
+ mc = memcacheclient.Client(["{}:{}".format(ba, bp)], debug=1)
+ rando = random.random()
+ mc.set("the_password", rando)
+ self.assertEquals(rando, mc.get("the_password"))
+ mc.disconnect_all()
+
+
+ def tearDown(self):
+ """
+ Verify that our spawned memcached can be reaped in no more than
+ one second - if not we'll get reactor unclean failures.
+ """
+ self.monitor.stopService()
+ sleep(1)
+ if os.path.exists(self.socket):
+ os.remove(self.socket)
+
+
class ProcessMonitorTests(CalDAVServiceMakerTestBase):
def configure(self):
Modified: CalendarServer/trunk/twistedcaldav/memcachepool.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcachepool.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/twistedcaldav/memcachepool.py 2015-03-31 20:20:48 UTC (rev 14640)
@@ -23,6 +23,7 @@
from twext.python.log import Logger
from twext.internet.gaiendpoint import GAIEndpoint
from twext.internet.adaptendpoint import connect
+from twisted.internet.endpoints import UNIXClientEndpoint
@@ -446,10 +447,15 @@
from twisted.internet import reactor
for name, pool in pools.items():
if pool["ClientEnabled"]:
+ if pool["MemcacheSocket"] is not '':
+ ep = UNIXClientEndpoint(reactor, pool["MemcacheSocket"])
+ else:
+ ep = GAIEndpoint(reactor, pool["BindAddress"], pool["Port"])
+
_installPool(
name,
pool["HandleCacheTypes"],
- GAIEndpoint(reactor, pool["BindAddress"], pool["Port"]),
+ ep,
maxClients,
reactor,
)
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-03-31 17:04:30 UTC (rev 14639)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-03-31 20:20:48 UTC (rev 14640)
@@ -932,6 +932,9 @@
"MaxClients": 5,
"Pools": {
"Default": {
+ # A unix socket used for communication with memcached.
+ # If blank, then an AF_INET socket is used instead.
+ "MemcacheSocket": "memcache.sock",
"ClientEnabled": True,
"ServerEnabled": True,
"BindAddress": "127.0.0.1",
@@ -1200,6 +1203,7 @@
("RunRoot", "PIDFile"),
("RunRoot", ("Stats", "UnixStatsSocket",)),
("RunRoot", "ControlSocket"),
+ ("RunRoot", ("Memcached", "Pools", "Default", "MemcacheSocket")),
("RunRoot", ("DirectoryProxy", "SocketPath",)),
]
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150331/9d5efaef/attachment-0001.html>
More information about the calendarserver-changes
mailing list