[CalendarServer-changes] [10032] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Nov 14 14:41:32 PST 2012
Revision: 10032
http://trac.calendarserver.org//changeset/10032
Author: sagen at apple.com
Date: 2012-11-14 14:41:32 -0800 (Wed, 14 Nov 2012)
Log Message:
-----------
Command gateway is now data-store enabled, and purge-old-events has been added as a command
Modified Paths:
--------------
CalendarServer/trunk/bin/calendarserver_command_gateway
CalendarServer/trunk/calendarserver/tools/gateway.py
CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
CalendarServer/trunk/twistedcaldav/test/util.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Modified: CalendarServer/trunk/bin/calendarserver_command_gateway
===================================================================
--- CalendarServer/trunk/bin/calendarserver_command_gateway 2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/bin/calendarserver_command_gateway 2012-11-14 22:41:32 UTC (rev 10032)
@@ -16,8 +16,13 @@
# limitations under the License.
##
+import os
import sys
+# In OS X Server context, add to PATH to find Postgres utilities (initdb, pg_ctl)
+if "Server.app" in sys.argv[0]:
+ os.environ["PATH"] += ":" + os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), "bin")
+
#PYTHONPATH
if __name__ == "__main__":
Modified: CalendarServer/trunk/calendarserver/tools/gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/gateway.py 2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/calendarserver/tools/gateway.py 2012-11-14 22:41:32 UTC (rev 10032)
@@ -17,28 +17,25 @@
##
from getopt import getopt, GetoptError
-from grp import getgrnam
-from pwd import getpwnam
import os
import sys
import xml
from twext.python.plistlib import readPlistFromString, writePlistToString
-from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
-from twisted.python.util import switchUID
-from twistedcaldav.config import config, ConfigurationError
from twistedcaldav.directory.directory import DirectoryError
from txdav.xml import element as davxml
-from calendarserver.tools.util import loadConfig, getDirectory, setupMemcached, checkDirectory
from calendarserver.tools.principals import (
principalForPrincipalID, proxySubprincipal, addProxy, removeProxy,
getProxies, setProxies, ProxyError, ProxyWarning, updateRecord
)
+from calendarserver.tools.purge import WorkerService, purgeOldEvents, DEFAULT_BATCH_SIZE, DEFAULT_RETAIN_DAYS
+from calendarserver.tools.cmdline import utilityMain
from twext.python.log import StandardIOObserver
+from pycalendar.datetime import PyCalendarDateTime
def usage(e=None):
@@ -60,6 +57,25 @@
sys.exit(0)
+class RunnerService(WorkerService):
+ """
+ A wrapper around Runner which uses utilityMain to get the store
+ """
+
+ commands = None
+
+ @inlineCallbacks
+ def doWork(self):
+ """
+ Create/run a Runner to execute the commands
+ """
+ rootResource = self.rootResource()
+ directory = rootResource.getDirectory()
+ runner = Runner(rootResource, directory, self._store, self.commands)
+ if runner.validate():
+ yield runner.run( )
+
+
def main():
try:
@@ -92,40 +108,7 @@
else:
raise NotImplementedError(opt)
- try:
- loadConfig(configFileName)
- # Create the DataRoot directory before shedding privileges
- if config.DataRoot.startswith(config.ServerRoot + os.sep):
- checkDirectory(
- config.DataRoot,
- "Data root",
- access=os.W_OK,
- create=(0750, config.UserName, config.GroupName),
- )
-
- # Shed privileges
- if config.UserName and config.GroupName and os.getuid() == 0:
- uid = getpwnam(config.UserName).pw_uid
- gid = getgrnam(config.GroupName).gr_gid
- switchUID(uid, uid, gid)
-
- os.umask(config.umask)
-
- # Configure memcached client settings prior to setting up resource
- # hierarchy (in getDirectory)
- setupMemcached(config)
-
- try:
- config.directory = getDirectory()
- except DirectoryError, e:
- respondWithError(str(e))
- return
-
- except ConfigurationError, e:
- respondWithError(str(e))
- return
-
#
# Read commands from stdin
#
@@ -143,17 +126,10 @@
else:
commands = [plist]
- runner = Runner(config.directory, commands)
- if not runner.validate():
- return
+ RunnerService.commands = commands
+ utilityMain(configFileName, RunnerService)
- #
- # Start the reactor
- #
- reactor.callLater(0, runner.run)
- reactor.run()
-
attrMap = {
'GeneratedUID' : { 'attr' : 'guid', },
'RealName' : { 'attr' : 'fullName', },
@@ -176,8 +152,10 @@
class Runner(object):
- def __init__(self, directory, commands):
+ def __init__(self, root, directory, store, commands):
+ self.root = root
self.dir = directory
+ self.store = store
self.commands = commands
def validate(self):
@@ -208,9 +186,6 @@
respondWithError("Command failed: '%s'" % (str(e),))
raise
- finally:
- reactor.stop()
-
# Locations
def command_getLocationList(self, command):
@@ -218,7 +193,6 @@
@inlineCallbacks
def command_createLocation(self, command):
-
kwargs = {}
for key, info in attrMap.iteritems():
if command.has_key(key):
@@ -233,7 +207,7 @@
readProxies = command.get("ReadProxies", None)
writeProxies = command.get("WriteProxies", None)
principal = principalForPrincipalID(record.guid, directory=self.dir)
- (yield setProxies(principal, readProxies, writeProxies))
+ (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
respondWithRecordsOfType(self.dir, command, "locations")
@@ -251,7 +225,8 @@
return
recordDict['AutoSchedule'] = principal.getAutoSchedule()
recordDict['AutoAcceptGroup'] = principal.getAutoAcceptGroup()
- recordDict['ReadProxies'], recordDict['WriteProxies'] = (yield getProxies(principal))
+ recordDict['ReadProxies'], recordDict['WriteProxies'] = (yield getProxies(principal,
+ directory=self.dir))
respond(command, recordDict)
command_getResourceAttributes = command_getLocationAttributes
@@ -279,7 +254,7 @@
readProxies = command.get("ReadProxies", None)
writeProxies = command.get("WriteProxies", None)
principal = principalForPrincipalID(record.guid, directory=self.dir)
- (yield setProxies(principal, readProxies, writeProxies))
+ (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
yield self.command_getLocationAttributes(command)
@@ -316,7 +291,7 @@
readProxies = command.get("ReadProxies", None)
writeProxies = command.get("WriteProxies", None)
principal = principalForPrincipalID(record.guid, directory=self.dir)
- (yield setProxies(principal, readProxies, writeProxies))
+ (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
respondWithRecordsOfType(self.dir, command, "resources")
@@ -343,7 +318,7 @@
readProxies = command.get("ReadProxies", None)
writeProxies = command.get("WriteProxies", None)
principal = principalForPrincipalID(record.guid, directory=self.dir)
- (yield setProxies(principal, readProxies, writeProxies))
+ (yield setProxies(principal, readProxies, writeProxies, directory=self.dir))
yield self.command_getResourceAttributes(command)
@@ -456,6 +431,23 @@
(yield respondWithProxies(self.dir, command, principal, "read"))
+ @inlineCallbacks
+ def command_purgeOldEvents(self, command):
+ """
+ Convert RetainDays from the command dictionary into a date, then purge
+ events older than that date.
+
+ @param command: the dictionary parsed from the plist read from stdin
+ @type command: C{dict}
+ """
+ retainDays = command.get("RetainDays", DEFAULT_RETAIN_DAYS)
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-retainDays)
+ eventCount = (yield purgeOldEvents(self.store, self.dir, self.root, cutoff, DEFAULT_BATCH_SIZE))
+ respond(command, {'EventsRemoved' : eventCount, "RetainDays" : retainDays})
+
+
@inlineCallbacks
def respondWithProxies(directory, command, principal, proxyType):
proxies = []
@@ -464,7 +456,7 @@
membersProperty = (yield subPrincipal.readProperty(davxml.GroupMemberSet, None))
if membersProperty.children:
for member in membersProperty.children:
- proxyPrincipal = principalForPrincipalID(str(member))
+ proxyPrincipal = principalForPrincipalID(str(member), directory=directory)
proxies.append(proxyPrincipal.record.guid)
respond(command, {
Modified: CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist 2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist 2012-11-14 22:41:32 UTC (rev 10032)
@@ -91,11 +91,11 @@
<!-- Log root -->
<key>LogRoot</key>
- <string>/var/log/caldavd</string>
+ <string>Logs</string>
<!-- Run root -->
<key>RunRoot</key>
- <string>/var/run</string>
+ <string>Logs/state</string>
<!-- Child aliases -->
<key>Aliases</key>
@@ -279,7 +279,7 @@
-->
<key>ProxyLoadFromFile</key>
- <string>conf/auth/proxies-test.xml</string>
+ <string></string>
<!--
Special principals
Modified: CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_gateway.py 2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/calendarserver/tools/test/test_gateway.py 2012-11-14 22:41:32 UTC (rev 10032)
@@ -277,6 +277,13 @@
results = yield self.runCommand(command_removeWriteProxy)
self.assertEquals(len(results["result"]["Proxies"]), 0)
+ @inlineCallbacks
+ def test_purgeOldEvents(self):
+ results = yield self.runCommand(command_purgeOldEvents)
+ self.assertEquals(results["result"]["EventsRemoved"], 0)
+ self.assertEquals(results["result"]["RetainDays"], 42)
+ results = yield self.runCommand(command_purgeOldEventsNoDays)
+ self.assertEquals(results["result"]["RetainDays"], 365)
command_addReadProxy = """<?xml version="1.0" encoding="UTF-8"?>
@@ -589,3 +596,25 @@
</dict>
</plist>
"""
+
+command_purgeOldEvents = """<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <key>command</key>
+ <string>purgeOldEvents</string>
+ <key>RetainDays</key>
+ <integer>42</integer>
+</dict>
+</plist>
+"""
+
+command_purgeOldEventsNoDays = """<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <key>command</key>
+ <string>purgeOldEvents</string>
+</dict>
+</plist>
+"""
Modified: CalendarServer/trunk/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/util.py 2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/twistedcaldav/test/util.py 2012-11-14 22:41:32 UTC (rev 10032)
@@ -45,7 +45,11 @@
from txdav.common.datastore.test.util import deriveQuota
from txdav.common.datastore.file import CommonDataStore
+from twext.python.log import Logger
+log = Logger()
+
+
__all__ = [
"featureUnimplemented",
"testUnimplemented",
@@ -633,6 +637,7 @@
self.input = inputData
self.output = []
self.error = []
+ self.terminated = False
def connectionMade(self):
@@ -655,14 +660,18 @@
"""
Some output was received on stderr.
"""
+ # Ignore the Postgres "NOTICE" output
+ if "NOTICE" in data:
+ return
+
self.error.append(data)
+
# Attempt to exit promptly if a traceback is displayed, so we don't
# deal with timeouts.
- lines = ''.join(self.error).split("\n")
- if len(lines) > 1:
- errorReportLine = lines[-2].split(": ", 1)
- if len(errorReportLine) == 2 and ' ' not in errorReportLine[0] and '\t' not in errorReportLine[0]:
- self.transport.signalProcess("TERM")
+ if "Traceback" in data and not self.terminated:
+ log.error("Terminating process due to output: %s" % (data,))
+ self.terminated = True
+ self.transport.signalProcess("TERM")
def processEnded(self, why):
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2012-11-14 22:15:21 UTC (rev 10031)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2012-11-14 22:41:32 UTC (rev 10032)
@@ -756,6 +756,10 @@
docRoot = config.DocumentRoot
+ if not os.path.exists(docRoot):
+ log.info("DocumentRoot (%s) doesn't exist; skipping migration" % (docRoot,))
+ return
+
versionFilePath = os.path.join(docRoot, ".calendarserver_version")
onDiskVersion = 0
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121114/16f48c36/attachment-0001.html>
More information about the calendarserver-changes
mailing list