[CalendarServer-changes] [10032] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Wed Nov 14 14:41:32 PST 2012


Revision: 10032
          http://trac.calendarserver.org//changeset/10032
Author:   sagen at apple.com
Date:     2012-11-14 14:41:32 -0800 (Wed, 14 Nov 2012)
Log Message:
-----------
Command gateway is now data-store enabled, and purge-old-events has been added as a command

Modified Paths:
--------------
    CalendarServer/trunk/bin/calendarserver_command_gateway
    CalendarServer/trunk/calendarserver/tools/gateway.py
    CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
    CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
    CalendarServer/trunk/twistedcaldav/test/util.py
    CalendarServer/trunk/twistedcaldav/upgrade.py

Modified: CalendarServer/trunk/bin/calendarserver_command_gateway
===================================================================
--- CalendarServer/trunk/bin/calendarserver_command_gateway	2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/bin/calendarserver_command_gateway	2012-11-14 22:41:32 UTC (rev 10032)
@@ -16,8 +16,13 @@
 # limitations under the License.
 ##
 
+import os
 import sys
 
+# In OS X Server context, add to PATH to find Postgres utilities (initdb, pg_ctl)
+if "Server.app" in sys.argv[0]:
+    os.environ["PATH"] += ":" + os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), "bin")
+
 #PYTHONPATH
 
 if __name__ == "__main__":

Modified: CalendarServer/trunk/calendarserver/tools/gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/gateway.py	2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/calendarserver/tools/gateway.py	2012-11-14 22:41:32 UTC (rev 10032)
@@ -17,28 +17,25 @@
 ##
 
 from getopt import getopt, GetoptError
-from grp import getgrnam
-from pwd import getpwnam
 import os
 import sys
 import xml
 
 from twext.python.plistlib import readPlistFromString, writePlistToString
 
-from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks
-from twisted.python.util import switchUID
-from twistedcaldav.config import config, ConfigurationError
 from twistedcaldav.directory.directory import DirectoryError
 from txdav.xml import element as davxml
 
-from calendarserver.tools.util import loadConfig, getDirectory, setupMemcached, checkDirectory
 from calendarserver.tools.principals import (
     principalForPrincipalID, proxySubprincipal, addProxy, removeProxy,
     getProxies, setProxies, ProxyError, ProxyWarning, updateRecord
 )
+from calendarserver.tools.purge import WorkerService, purgeOldEvents, DEFAULT_BATCH_SIZE, DEFAULT_RETAIN_DAYS
+from calendarserver.tools.cmdline import utilityMain
 
 from twext.python.log import StandardIOObserver
+from pycalendar.datetime import PyCalendarDateTime
 
 
 def usage(e=None):
@@ -60,6 +57,25 @@
         sys.exit(0)
 
 
+class RunnerService(WorkerService):
+    """
+    A wrapper around Runner which uses utilityMain to get the store
+    """
+
+    commands = None
+
+    @inlineCallbacks
+    def doWork(self):
+        """
+        Create/run a Runner to execute the commands
+        """
+        rootResource = self.rootResource()
+        directory = rootResource.getDirectory()
+        runner = Runner(rootResource, directory, self._store, self.commands)
+        if runner.validate():
+            yield runner.run( )
+
+
 def main():
 
     try:
@@ -92,40 +108,7 @@
         else:
             raise NotImplementedError(opt)
 
-    try:
-        loadConfig(configFileName)
 
-        # Create the DataRoot directory before shedding privileges
-        if config.DataRoot.startswith(config.ServerRoot + os.sep):
-            checkDirectory(
-                config.DataRoot,
-                "Data root",
-                access=os.W_OK,
-                create=(0750, config.UserName, config.GroupName),
-            )
-
-        # Shed privileges
-        if config.UserName and config.GroupName and os.getuid() == 0:
-            uid = getpwnam(config.UserName).pw_uid
-            gid = getgrnam(config.GroupName).gr_gid
-            switchUID(uid, uid, gid)
-
-        os.umask(config.umask)
-
-        # Configure memcached client settings prior to setting up resource
-        # hierarchy (in getDirectory)
-        setupMemcached(config)
-
-        try:
-            config.directory = getDirectory()
-        except DirectoryError, e:
-            respondWithError(str(e))
-            return
-
-    except ConfigurationError, e:
-        respondWithError(str(e))
-        return
-
     #
     # Read commands from stdin
     #
@@ -143,17 +126,10 @@
     else:
         commands = [plist]
 
-    runner = Runner(config.directory, commands)
-    if not runner.validate():
-        return
+    RunnerService.commands = commands
+    utilityMain(configFileName, RunnerService)
 
-    #
-    # Start the reactor
-    #
-    reactor.callLater(0, runner.run)
-    reactor.run()
 
-
 attrMap = {
     'GeneratedUID' : { 'attr' : 'guid', },
     'RealName' : { 'attr' : 'fullName', },
@@ -176,8 +152,10 @@
 
 class Runner(object):
 
-    def __init__(self, directory, commands):
+    def __init__(self, root, directory, store, commands):
+        self.root = root
         self.dir = directory
+        self.store = store
         self.commands = commands
 
     def validate(self):
@@ -208,9 +186,6 @@
             respondWithError("Command failed: '%s'" % (str(e),))
             raise
 
-        finally:
-            reactor.stop()
-
     # Locations
 
     def command_getLocationList(self, command):
@@ -218,7 +193,6 @@
 
     @inlineCallbacks
     def command_createLocation(self, command):
-
         kwargs = {}
         for key, info in attrMap.iteritems():
             if command.has_key(key):
@@ -233,7 +207,7 @@
         readProxies = command.get("ReadProxies", None)
         writeProxies = command.get("WriteProxies", None)
         principal = principalForPrincipalID(record.guid, directory=self.dir)
-        (yield setProxies(principal, readProxies, writeProxies))
+        (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
 
         respondWithRecordsOfType(self.dir, command, "locations")
 
@@ -251,7 +225,8 @@
             return
         recordDict['AutoSchedule'] = principal.getAutoSchedule()
         recordDict['AutoAcceptGroup'] = principal.getAutoAcceptGroup()
-        recordDict['ReadProxies'], recordDict['WriteProxies'] = (yield getProxies(principal))
+        recordDict['ReadProxies'], recordDict['WriteProxies'] = (yield getProxies(principal,
+            directory=self.dir))
         respond(command, recordDict)
 
     command_getResourceAttributes = command_getLocationAttributes
@@ -279,7 +254,7 @@
         readProxies = command.get("ReadProxies", None)
         writeProxies = command.get("WriteProxies", None)
         principal = principalForPrincipalID(record.guid, directory=self.dir)
-        (yield setProxies(principal, readProxies, writeProxies))
+        (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
 
         yield self.command_getLocationAttributes(command)
 
@@ -316,7 +291,7 @@
         readProxies = command.get("ReadProxies", None)
         writeProxies = command.get("WriteProxies", None)
         principal = principalForPrincipalID(record.guid, directory=self.dir)
-        (yield setProxies(principal, readProxies, writeProxies))
+        (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
 
         respondWithRecordsOfType(self.dir, command, "resources")
 
@@ -343,7 +318,7 @@
         readProxies = command.get("ReadProxies", None)
         writeProxies = command.get("WriteProxies", None)
         principal = principalForPrincipalID(record.guid, directory=self.dir)
-        (yield setProxies(principal, readProxies, writeProxies))
+        (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
 
         yield self.command_getResourceAttributes(command)
 
@@ -456,6 +431,23 @@
         (yield respondWithProxies(self.dir, command, principal, "read"))
 
 
+    @inlineCallbacks
+    def command_purgeOldEvents(self, command):
+        """
+        Convert RetainDays from the command dictionary into a date, then purge
+        events older than that date.
+
+        @param command: the dictionary parsed from the plist read from stdin
+        @type command: C{dict}
+        """
+        retainDays = command.get("RetainDays", DEFAULT_RETAIN_DAYS)
+        cutoff = PyCalendarDateTime.getToday()
+        cutoff.setDateOnly(False)
+        cutoff.offsetDay(-retainDays)
+        eventCount = (yield purgeOldEvents(self.store, self.dir, self.root, cutoff, DEFAULT_BATCH_SIZE))
+        respond(command, {'EventsRemoved' : eventCount, "RetainDays" : retainDays})
+
+
 @inlineCallbacks
 def respondWithProxies(directory, command, principal, proxyType):
     proxies = []
@@ -464,7 +456,7 @@
         membersProperty = (yield subPrincipal.readProperty(davxml.GroupMemberSet, None))
         if membersProperty.children:
             for member in membersProperty.children:
-                proxyPrincipal = principalForPrincipalID(str(member))
+                proxyPrincipal = principalForPrincipalID(str(member), directory=directory)
                 proxies.append(proxyPrincipal.record.guid)
 
     respond(command, {

Modified: CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist	2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist	2012-11-14 22:41:32 UTC (rev 10032)
@@ -91,11 +91,11 @@
 
     <!-- Log root -->
     <key>LogRoot</key>
-    <string>/var/log/caldavd</string>
+    <string>Logs</string>
 
     <!-- Run root -->
     <key>RunRoot</key>
-    <string>/var/run</string>
+    <string>Logs/state</string>
 
     <!-- Child aliases -->
     <key>Aliases</key>
@@ -279,7 +279,7 @@
      -->
 
 	<key>ProxyLoadFromFile</key>
-    <string>conf/auth/proxies-test.xml</string>
+    <string></string>
 
     <!--
         Special principals

Modified: CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_gateway.py	2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/calendarserver/tools/test/test_gateway.py	2012-11-14 22:41:32 UTC (rev 10032)
@@ -277,6 +277,13 @@
         results = yield self.runCommand(command_removeWriteProxy)
         self.assertEquals(len(results["result"]["Proxies"]), 0)
 
+    @inlineCallbacks
+    def test_purgeOldEvents(self):
+        results = yield self.runCommand(command_purgeOldEvents)
+        self.assertEquals(results["result"]["EventsRemoved"], 0)
+        self.assertEquals(results["result"]["RetainDays"], 42)
+        results = yield self.runCommand(command_purgeOldEventsNoDays)
+        self.assertEquals(results["result"]["RetainDays"], 365)
 
 
 command_addReadProxy = """<?xml version="1.0" encoding="UTF-8"?>
@@ -589,3 +596,25 @@
 </dict>
 </plist>
 """
+
+command_purgeOldEvents = """<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+        <key>command</key>
+        <string>purgeOldEvents</string>
+        <key>RetainDays</key>
+        <integer>42</integer>
+</dict>
+</plist>
+"""
+
+command_purgeOldEventsNoDays = """<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+        <key>command</key>
+        <string>purgeOldEvents</string>
+</dict>
+</plist>
+"""

Modified: CalendarServer/trunk/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/util.py	2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/twistedcaldav/test/util.py	2012-11-14 22:41:32 UTC (rev 10032)
@@ -45,7 +45,11 @@
 from txdav.common.datastore.test.util import deriveQuota
 from txdav.common.datastore.file import CommonDataStore
 
+from twext.python.log import Logger
 
+log = Logger()
+
+
 __all__ = [
     "featureUnimplemented",
     "testUnimplemented",
@@ -633,6 +637,7 @@
         self.input = inputData
         self.output = []
         self.error = []
+        self.terminated = False
 
 
     def connectionMade(self):
@@ -655,14 +660,18 @@
         """
         Some output was received on stderr.
         """
+        # Ignore the Postgres "NOTICE" output
+        if "NOTICE" in data:
+            return
+
         self.error.append(data)
+
         # Attempt to exit promptly if a traceback is displayed, so we don't
         # deal with timeouts.
-        lines = ''.join(self.error).split("\n")
-        if len(lines) > 1:
-            errorReportLine = lines[-2].split(": ", 1)
-            if len(errorReportLine) == 2 and ' ' not in errorReportLine[0] and '\t' not in errorReportLine[0]:
-                self.transport.signalProcess("TERM")
+        if "Traceback" in data and not self.terminated:
+            log.error("Terminating process due to output: %s" % (data,))
+            self.terminated = True
+            self.transport.signalProcess("TERM")
 
 
     def processEnded(self, why):

Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py	2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py	2012-11-14 22:41:32 UTC (rev 10032)
@@ -756,6 +756,10 @@
 
     docRoot = config.DocumentRoot
 
+    if not os.path.exists(docRoot):
+        log.info("DocumentRoot (%s) doesn't exist; skipping migration" % (docRoot,))
+        return
+
     versionFilePath = os.path.join(docRoot, ".calendarserver_version")
 
     onDiskVersion = 0
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121114/16f48c36/attachment-0001.html>


More information about the calendarserver-changes mailing list