[CalendarServer-changes] [4354] CalendarServer/branches/users/wsanchez/deployment
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jun 17 14:52:39 PDT 2009
Revision: 4354
http://trac.macosforge.org/projects/calendarserver/changeset/4354
Author: wsanchez at apple.com
Date: 2009-06-17 14:52:39 -0700 (Wed, 17 Jun 2009)
Log Message:
-----------
Update memcacheclient code to latest.
Update config.py so we don't have to keep converting dict-style usage to attr-style usage.
Modified Paths:
--------------
CalendarServer/branches/users/wsanchez/deployment/memcacheclient.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/config.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/memcacheprops.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_config.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/util.py
Added Paths:
-----------
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_memcacheprops.py
Modified: CalendarServer/branches/users/wsanchez/deployment/memcacheclient.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/memcacheclient.py 2009-06-17 18:51:12 UTC (rev 4353)
+++ CalendarServer/branches/users/wsanchez/deployment/memcacheclient.py 2009-06-17 21:52:39 UTC (rev 4354)
@@ -13,8 +13,8 @@
This should give you a feel for how this module operates::
- import memcache
- mc = memcache.Client(['127.0.0.1:11211'], debug=0)
+ import memcacheclient
+ mc = memcacheclient.Client(['127.0.0.1:11211'], debug=0)
mc.set("some_key", "Some value")
value = mc.get("some_key")
@@ -49,6 +49,12 @@
import os
import re
import types
+from twistedcaldav.config import config
+from twistedcaldav.log import Logger
+
+log = Logger()
+
+
try:
import cPickle as pickle
except ImportError:
@@ -78,7 +84,7 @@
SERVER_MAX_KEY_LENGTH = 250
# Storing values larger than 1MB requires recompiling memcached. If you do,
-# this value can be changed by doing "memcache.SERVER_MAX_VALUE_LENGTH = N"
+# this value can be changed by doing "memcacheclient.SERVER_MAX_VALUE_LENGTH = N"
# after importing this module.
SERVER_MAX_VALUE_LENGTH = 1024*1024
@@ -95,6 +101,11 @@
NOT_FOUND error
"""
+class TokenMismatchError(MemcacheError):
+ """
+ Check-and-set token mismatch
+ """
+
try:
# Only exists in Python 2.4+
from threading import local
@@ -103,7 +114,27 @@
class local(object):
pass
+class ClientFactory(object):
+ # unit tests should set this to True to enable the fake test cache
+ allowTestCache = False
+
+ @classmethod
+ def getClient(cls, servers, debug=0, pickleProtocol=0,
+ pickler=pickle.Pickler, unpickler=pickle.Unpickler,
+ pload=None, pid=None):
+
+ if config.Memcached.ClientEnabled:
+ return Client(servers, debug=debug, pickleProtocol=pickleProtocol,
+ pickler=pickler, unpickler=unpickler, pload=pload, pid=pid)
+ elif cls.allowTestCache:
+ return TestClient(servers, debug=debug,
+ pickleProtocol=pickleProtocol, pickler=pickler,
+ unpickler=unpickler, pload=pload, pid=pid)
+ else:
+ return None
+
+
class Client(local):
"""
Object representing a pool of memcache servers.
@@ -289,6 +320,7 @@
#print "(using server %s)" % server,
return server, key
serverhash = serverHashFunction(str(serverhash) + str(i))
+ log.error("Memcacheclient _get_server( ) failed to connect")
return None, None
def disconnect_all(self):
@@ -692,16 +724,29 @@
if not store_info: return(0)
if token is not None:
+ cmd = "cas"
fullcmd = "cas %s %d %d %d %s\r\n%s" % (key, store_info[0], time, store_info[1], token, store_info[2])
else:
fullcmd = "%s %s %d %d %d\r\n%s" % (cmd, key, store_info[0], time, store_info[1], store_info[2])
try:
server.send_cmd(fullcmd)
result = server.expect("STORED")
+
+ if (result == "STORED"):
+ return True
+
if (result == "NOT_FOUND"):
raise NotFoundError(key)
- return (result == "STORED")
+ if token and result == "EXISTS":
+ log.debug("Memcacheclient check-and-set failed")
+ raise TokenMismatchError(key)
+
+ log.error("Memcacheclient %s command failed with result (%s)" %
+ (cmd, result))
+
+ return False
+
except socket.error, msg:
if type(msg) is types.TupleType: msg = msg[1]
server.mark_dead(msg)
@@ -929,6 +974,157 @@
return val
+
+class TestClient(Client):
+ """
+ Fake memcache client for unit tests
+
+ """
+
+ def __init__(self, servers, debug=0, pickleProtocol=0,
+ pickler=pickle.Pickler, unpickler=pickle.Unpickler,
+ pload=None, pid=None):
+
+ local.__init__(self)
+
+ super(TestClient, self).__init__(servers, debug=debug,
+ pickleProtocol=pickleProtocol, pickler=pickler, unpickler=unpickler,
+ pload=pload, pid=pid)
+
+ self.data = {}
+ self.token = 0
+
+
+
+ def get_stats(self):
+ raise NotImplementedError()
+
+ def get_slabs(self):
+ raise NotImplementedError()
+
+ def flush_all(self):
+ raise NotImplementedError()
+
+ def forget_dead_hosts(self):
+ raise NotImplementedError()
+
+ def delete_multi(self, keys, time=0, key_prefix=''):
+ '''
+ Delete multiple keys in the memcache doing just one query.
+
+ >>> notset_keys = mc.set_multi({'key1' : 'val1', 'key2' : 'val2'})
+ >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1', 'key2' : 'val2'}
+ 1
+ >>> mc.delete_multi(['key1', 'key2'])
+ 1
+ >>> mc.get_multi(['key1', 'key2']) == {}
+ 1
+ '''
+
+ self._statlog('delete_multi')
+ for key in keys:
+ key = key_prefix + key
+ del self.data[key]
+ return 1
+
+ def delete(self, key, time=0):
+ '''Deletes a key from the memcache.
+
+ @return: Nonzero on success.
+ @param time: number of seconds any subsequent set / update commands should fail. Defaults to 0 for no delay.
+ @rtype: int
+ '''
+ check_key(key)
+ del self.data[key]
+ return 1
+
+
+ def incr(self, key, delta=1):
+ raise NotImplementedError()
+
+ def decr(self, key, delta=1):
+ raise NotImplementedError()
+
+ def add(self, key, val, time = 0, min_compress_len = 0):
+ raise NotImplementedError()
+
+ def append(self, key, val, time=0, min_compress_len=0):
+ raise NotImplementedError()
+
+ def prepend(self, key, val, time=0, min_compress_len=0):
+ raise NotImplementedError()
+
+ def replace(self, key, val, time=0, min_compress_len=0):
+ raise NotImplementedError()
+
+ def set(self, key, val, time=0, min_compress_len=0, token=None):
+ self._statlog('set')
+ return self._set("set", key, val, time, min_compress_len, token=token)
+
+ def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0):
+ self._statlog('set_multi')
+ for key, val in mapping.iteritems():
+ key = key_prefix + key
+ self._set("set", key, val, time, min_compress_len)
+ return []
+
+ def _set(self, cmd, key, val, time, min_compress_len = 0, token=None):
+ check_key(key)
+ self._statlog(cmd)
+
+ serialized = pickle.dumps(val, pickle.HIGHEST_PROTOCOL)
+
+ if token is not None:
+ if self.data.has_key(key):
+ stored_val, stored_token = self.data[key]
+ if token != stored_token:
+ raise TokenMismatchError(key)
+
+ self.data[key] = (serialized, str(self.token))
+ self.token += 1
+
+ return True
+
+ def get(self, key):
+ check_key(key)
+
+ self._statlog('get')
+ if self.data.has_key(key):
+ stored_val, stored_token = self.data[key]
+ val = pickle.loads(stored_val)
+ return val
+ return None
+
+
+ def gets(self, key):
+ check_key(key)
+ if self.data.has_key(key):
+ stored_val, stored_token = self.data[key]
+ val = pickle.loads(stored_val)
+ return (val, stored_token)
+ return (None, None)
+
+ def get_multi(self, keys, key_prefix=''):
+ self._statlog('get_multi')
+
+ results = {}
+ for key in keys:
+ key = key_prefix + key
+ val = self.get(key)
+ results[key] = val
+ return results
+
+ def gets_multi(self, keys, key_prefix=''):
+ self._statlog('gets_multi')
+ results = {}
+ for key in keys:
+ key = key_prefix + key
+ result = self.gets(key)
+ if result[1] is not None:
+ results[key] = result
+ return results
+
+
class _Host:
_DEAD_RETRY = 1 # number of seconds before retrying a dead server.
_SOCKET_TIMEOUT = 3 # number of seconds before sockets timeout.
@@ -979,12 +1175,14 @@
return 0
def mark_dead(self, reason):
+ log.error("Memcacheclient socket marked dead (%s)" % (reason,))
self.debuglog("MemCache: %s: %s. Marking dead." % (self, reason))
self.deaduntil = time.time() + _Host._DEAD_RETRY
self.close_socket()
def _get_socket(self):
if self._check_dead():
+ log.error("Memcacheclient _get_socket() found dead socket")
return None
if self.socket:
return self.socket
@@ -993,10 +1191,14 @@
try:
s.connect(self.address)
except socket.timeout, msg:
+ log.error("Memcacheclient _get_socket() connection timed out (%s)" %
+ (msg,))
self.mark_dead("connect: %s" % msg)
return None
except socket.error, msg:
if type(msg) is types.TupleType: msg = msg[1]
+ log.error("Memcacheclient _get_socket() connection error (%s)" %
+ (msg,))
self.mark_dead("connect: %s" % msg[1])
return None
self.socket = s
@@ -1091,11 +1293,11 @@
raise Client.MemcachedKeyCharacterError, "Control characters not allowed"
def _doctest():
- import doctest, memcache
+ import doctest, memcacheclient
servers = ["127.0.0.1:11211"]
mc = Client(servers, debug=1)
globs = {"mc": mc}
- return doctest.testmod(memcache, globs=globs)
+ return doctest.testmod(memcacheclient, globs=globs)
if __name__ == "__main__":
print "Testing docstrings..."
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/config.py 2009-06-17 18:51:12 UTC (rev 4353)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/config.py 2009-06-17 21:52:39 UTC (rev 4354)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,6 +14,15 @@
# limitations under the License.
##
+__all__ = [
+ "defaultConfigFile",
+ "defaultConfig",
+ "ConfigDict",
+ "Config",
+ "ConfigurationError",
+ "config",
+]
+
import os
import copy
import re
@@ -27,6 +36,33 @@
log = Logger()
+class ConfigDict (dict):
+ def __init__(self, mapping=None):
+ if mapping is not None:
+ for key, value in mapping.iteritems():
+ self[key] = value
+
+ def __repr__(self):
+ return "*" + dict.__repr__(self)
+
+ def __setitem__(self, key, value):
+ if isinstance(value, dict) and not isinstance(value, self.__class__):
+ dict.__setitem__(self, key, self.__class__(value))
+ else:
+ dict.__setitem__(self, key, value)
+
+ def __setattr__(self, attr, value):
+ if attr[0] == "_":
+ dict.__setattr__(self, attr, value)
+ else:
+ self[attr] = value
+
+ def __getattr__(self, attr):
+ if attr in self:
+ return self[attr]
+ else:
+ return dict.__getattr__(self, attr)
+
defaultConfigFile = "/etc/caldavd/caldavd.plist"
serviceDefaultParams = {
@@ -41,6 +77,8 @@
}
defaultConfig = {
+ # Note: Don't use None values below; that confuses the command-line parser.
+
#
# Public network address information
#
@@ -54,8 +92,6 @@
"HTTPPort": 0, # HTTP port (0 to disable HTTP)
"SSLPort" : 0, # SSL port (0 to disable HTTPS)
- # Note: we'd use None above, but that confuses the command-line parser.
-
#
# Network address configuration information
#
@@ -117,7 +153,7 @@
},
"Kerberos": { # Kerberos/SPNEGO
"Enabled": False,
- "ServicePrincipal": ''
+ "ServicePrincipal": ""
},
},
@@ -129,21 +165,22 @@
"ServerStatsFile": "/var/run/caldavd/stats.plist",
"PIDFile" : "/var/run/caldavd.pid",
"RotateAccessLog" : False,
- "MoreAccessLogData" : False,
+ "MoreAccessLogData" : True,
"DefaultLogLevel" : "",
"LogLevels" : {},
+
"AccountingCategories": {
"iTIP": False,
},
"AccountingPrincipals": [],
- "AccountingLogRoot": "/var/log/caldavd/accounting",
+ "AccountingLogRoot" : "/var/log/caldavd/accounting",
#
# SSL/TLS
#
- "SSLCertificate": "/etc/certificates/Default.crt", # Public key
- "SSLPrivateKey": "/etc/certificates/Default.key", # Private key
- "SSLAuthorityChain": "", # Certificate Authority Chain
+ "SSLCertificate" : "", # Public key
+ "SSLPrivateKey" : "", # Private key
+ "SSLAuthorityChain" : "", # Certificate Authority Chain
"SSLPassPhraseDialog": "/etc/apache2/getsslpassphrase",
#
@@ -179,24 +216,24 @@
"EnableDropBox" : False, # Calendar Drop Box
"EnablePrivateEvents" : False, # Private Events
"EnableTimezoneService" : False, # Timezone service
- "EnableAutoAcceptTrigger" : False, # Manually trigger auto-accept behavior
#
# Notifications
#
"Notifications" : {
"Enabled": False,
- "CoalesceSeconds" : 10,
+ "CoalesceSeconds" : 3,
"InternalNotificationHost" : "localhost",
"InternalNotificationPort" : 62309,
+ "BindAddress" : "127.0.0.1",
- "Services" : [
- {
+ "Services" : {
+ "SimpleLineNotifier" : {
"Service" : "twistedcaldav.notify.SimpleLineNotifierService",
"Enabled" : False,
"Port" : 62308,
},
- {
+ "XMPPNotifier" : {
"Service" : "twistedcaldav.notify.XMPPNotifierService",
"Enabled" : False,
"Host" : "", # "xmpp.host.name"
@@ -204,13 +241,40 @@
"JID" : "", # "jid at xmpp.host.name/resource"
"Password" : "",
"ServiceAddress" : "", # "pubsub.xmpp.host.name"
+ "NodeConfiguration" : {
+ "pubsub#deliver_payloads" : "1",
+ "pubsub#persist_items" : "1",
+ },
"KeepAliveSeconds" : 120,
- "TestJID": "",
+ "HeartbeatMinutes" : 30,
+ "AllowedJIDs": [],
},
- ]
+ }
},
#
+ # Performance tuning
+ #
+
+ # Set the maximum number of outstanding requests to this server.
+ "MaxRequests": 600,
+
+ "ListenBacklog": 50,
+ "IdleConnectionTimeOut": 15,
+ "UIDReservationTimeOut": 30 * 60,
+
+
+ #
+ # Localization
+ #
+ "Localization" : {
+ "TranslationsDirectory" : "/usr/share/caldavd/share/translations",
+ "LocalesDirectory" : "/usr/share/caldavd/share/locales",
+ "Language" : "English",
+ },
+
+
+ #
# Implementation details
#
# The following are specific to how the server is built, and useful
@@ -234,34 +298,27 @@
"umask": 0027,
# A unix socket used for communication between the child and master
- # processes.
+ # processes. If blank, then an AF_INET socket is used instead.
"ControlSocket": "/var/run/caldavd.sock",
+
# Support for Content-Encoding compression options as specified in
# RFC2616 Section 3.5
"ResponseCompression": True,
- # The retry-after value (in seconds) to return with a 503 error
+ # The retry-after value (in seconds) to return with a 503 error
"HTTPRetryAfter": 180,
- # Set the maximum number of outstanding requests to this server.
- "MaxRequests": 600,
-
- # Configure the number of seconds that Propfinds should be cached for.
- "ResponseCacheSize": 1000,
-
# Profiling options
"Profiling": {
"Enabled": False,
"BaseDirectory": "/tmp/stats",
},
- "ListenBacklog": 50,
-
"Memcached": {
"MaxClients": 5,
- "ClientEnabled": False,
- "ServerEnabled": False,
+ "ClientEnabled": True,
+ "ServerEnabled": True,
"BindAddress": "127.0.0.1",
"Port": 11211,
"memcached": "memcached", # Find in PATH
@@ -270,17 +327,23 @@
},
"EnableKeepAlive": True,
- "IdleConnectionTimeOut": 15,
- "UIDReservationTimeOut": 30 * 60
+
+ "ResponseCacheTimeout": 30, # Minutes
}
-
class Config (object):
+ """
+ @DynamicAttrs
+ """
def __init__(self, defaults):
+ if not isinstance(defaults, ConfigDict):
+ defaults = ConfigDict(defaults)
+
self.setDefaults(defaults)
- self._data = copy.deepcopy(self._defaults)
+ self._data = copy.deepcopy(defaults)
self._configFile = None
self._hooks = [
+ self.updateHostName,
self.updateDirectoryService,
self.updateACLs,
self.updateRejectClients,
@@ -296,6 +359,9 @@
self._hooks.append(hook)
def update(self, items):
+ if not isinstance(items, ConfigDict):
+ items = ConfigDict(items)
+
#
# Call hooks
#
@@ -303,33 +369,43 @@
hook(self, items)
@staticmethod
+ def updateHostName(self, items):
+ if not self.ServerHostName:
+ from socket import getfqdn
+ hostname = getfqdn()
+ if not hostname:
+ hostname = "localhost"
+ self.ServerHostName = hostname
+
+ @staticmethod
def updateDirectoryService(self, items):
#
# Special handling for directory services configs
#
dsType = items.get("DirectoryService", {}).get("type", None)
if dsType is None:
- dsType = self._data["DirectoryService"]["type"]
+ dsType = self._data.DirectoryService.type
else:
- if dsType == self._data["DirectoryService"]["type"]:
- oldParams = self._data["DirectoryService"]["params"]
- newParams = items["DirectoryService"].get("params", {})
+ if dsType == self._data.DirectoryService.type:
+ oldParams = self._data.DirectoryService.params
+ newParams = items.DirectoryService.get("params", {})
_mergeData(oldParams, newParams)
else:
if dsType in serviceDefaultParams:
- self._data["DirectoryService"]["params"] = copy.deepcopy(serviceDefaultParams[dsType])
+ self._data.DirectoryService.params = copy.deepcopy(serviceDefaultParams[dsType])
else:
- self._data["DirectoryService"]["params"] = {}
+ self._data.DirectoryService.params = {}
for param in items.get("DirectoryService", {}).get("params", {}):
- if param not in serviceDefaultParams[dsType]:
- raise ConfigurationError("Parameter %s is not supported by service %s" % (param, dsType))
+ if dsType in serviceDefaultParams and param not in serviceDefaultParams[dsType]:
+ log.warn("Parameter %s is not supported by service %s" % (param, dsType))
_mergeData(self._data, items)
- for param in tuple(self._data["DirectoryService"]["params"]):
- if param not in serviceDefaultParams[self._data["DirectoryService"]["type"]]:
- del self._data["DirectoryService"]["params"][param]
+ if self._data.DirectoryService.type in serviceDefaultParams:
+ for param in tuple(self._data.DirectoryService.params):
+ if param not in serviceDefaultParams[self._data.DirectoryService.type]:
+ del self._data.DirectoryService.params[param]
@staticmethod
def updateACLs(self, items):
@@ -431,7 +507,7 @@
if "DefaultLogLevel" in self._data:
level = self._data["DefaultLogLevel"]
if not level:
- level = "info"
+ level = "warn"
setLogLevelForNamespace(None, level)
if "LogLevels" in self._data:
@@ -446,6 +522,8 @@
self.update(items)
def setDefaults(self, defaults):
+ if not isinstance(defaults, ConfigDict):
+ defaults = ConfigDict(defaults)
self._defaults = copy.deepcopy(defaults)
def __setattr__(self, attr, value):
@@ -468,32 +546,48 @@
def loadConfig(self, configFile):
self._configFile = configFile
- if configFile and os.path.exists(configFile):
- configDict = readPlist(configFile)
- configDict = _cleanup(configDict)
- self.update(configDict)
- elif configFile:
- log.error("Configuration file does not exist or is inaccessible: %s" % (configFile,))
+ if configFile:
+ try:
+ configDict = readPlist(configFile)
+ except (IOError, OSError):
+ log.error("Unable to open config file: %s" % (configFile,))
+ else:
+ configDict = _cleanup(configDict)
+ self.update(ConfigDict(configDict))
@staticmethod
def updateNotifications(self, items):
#
# Notifications
#
- for service in self.Notifications["Services"]:
+ for key, service in self.Notifications["Services"].iteritems():
if service["Enabled"]:
self.Notifications["Enabled"] = True
break
else:
self.Notifications["Enabled"] = False
- for service in self.Notifications["Services"]:
+ for key, service in self.Notifications["Services"].iteritems():
if (
service["Service"] == "twistedcaldav.notify.XMPPNotifierService" and
service["Enabled"]
):
+ # Get password from keychain. If not there, fall back to what
+ # is in the plist.
+ try:
+ password = getPasswordFromKeychain(service["JID"])
+ service["Password"] = password
+ log.info("XMPP password successfully retreived from keychain")
+ except KeychainAccessError:
+ # The system doesn't support keychain
+ pass
+ except KeychainPasswordNotFound:
+ # The password doesn't exist in the keychain.
+ log.error("XMPP password not found in keychain")
+
+ # Check for empty fields
for key, value in service.iteritems():
- if not value and key not in ("TestJID"):
+ if not value and key not in ("AllowedJIDs", "HeartbeatMinutes", "Password"):
raise ConfigurationError("Invalid %s for XMPPNotifierService: %r"
% (key, value))
@@ -502,7 +596,7 @@
for key, value in newData.iteritems():
if isinstance(value, (dict,)):
if key in oldData:
- assert isinstance(oldData[key], (dict,))
+ assert isinstance(oldData[key], ConfigDict), "%r in %r is not a ConfigDict" % (oldData[key], oldData)
else:
oldData[key] = {}
_mergeData(oldData[key], value)
@@ -518,6 +612,11 @@
def deprecated(oldKey, newKey):
log.err("Configuration option %r is deprecated in favor of %r." % (oldKey, newKey))
+ if oldKey in configDict and newKey in configDict:
+ raise ConfigurationError(
+ "Both %r and %r options are specified; use the %r option only."
+ % (oldKey, newKey, newKey)
+ )
def renamed(oldKey, newKey):
deprecated(oldKey, newKey)
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/memcacheprops.py 2009-06-17 18:51:12 UTC (rev 4353)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/memcacheprops.py 2009-06-17 21:52:39 UTC (rev 4354)
@@ -32,7 +32,7 @@
except ImportError:
from md5 import new as md5
-from memcacheclient import Client as MemcacheClient, MemcacheError
+from memcacheclient import ClientFactory as MemcacheClientFactory, MemcacheError, TokenMismatchError
from twisted.python.filepath import FilePath
from twisted.web2 import responsecode
@@ -45,6 +45,7 @@
NoValue = ""
+
class MemcachePropertyCollection (LoggingMixIn):
"""
Manages a single property store for all resources in a collection.
@@ -56,12 +57,10 @@
@classmethod
def memcacheClient(cls, refresh=False):
if not hasattr(MemcachePropertyCollection, "_memcacheClient"):
- if not config.Memcached["ClientEnabled"]:
- return None
log.info("Instantiating memcache connection for MemcachePropertyCollection")
- MemcachePropertyCollection._memcacheClient = MemcacheClient(
- ["%s:%s" % (config.Memcached["BindAddress"], config.Memcached["Port"])],
+
+ MemcachePropertyCollection._memcacheClient = MemcacheClientFactory.getClient(["%s:%s" % (config.Memcached.BindAddress, config.Memcached.Port)],
debug=0,
pickleProtocol=2,
)
@@ -200,25 +199,57 @@
return cache
- def setProperty(self, child, property):
+ def setProperty(self, child, property, delete=False):
propertyCache, key, childCache, token = self.childCache(child)
- if childCache.get(property.qname(), None) == property:
- # No changes
- return
+ if delete:
+ qname = property
+ if childCache.has_key(qname):
+ del childCache[qname]
+ else:
+ qname = property.qname()
+ childCache[qname] = property
- childCache[property.qname()] = property
+ client = self.memcacheClient()
- client = self.memcacheClient()
if client is not None:
- result = client.set(key, childCache, time=self.cacheTimeout, token=token)
- if not result:
+ retries = 10
+ while retries:
+ try:
+ if client.set(key, childCache, time=self.cacheTimeout,
+ token=token):
+ # Success
+ break
+
+ except TokenMismatchError:
+ # The value in memcache has changed since we last
+ # fetched it
+ log.debug("memcacheprops setProperty TokenMismatchError; retrying...")
+
+ finally:
+ # Re-fetch the properties for this child
+ loaded = self._loadCache(childNames=(child.fp.basename(),))
+ propertyCache.update(loaded.iteritems())
+
+ retries -= 1
+
+ propertyCache, key, childCache, token = self.childCache(child)
+
+ if delete:
+ if childCache.has_key(qname):
+ del childCache[qname]
+ else:
+ childCache[qname] = property
+
+ else:
+ log.error("memcacheprops setProperty had too many failures")
delattr(self, "_propertyCache")
- raise MemcacheError("Unable to set property %s on %s"
- % (property.sname(), child))
+ raise MemcacheError("Unable to %s property {%s}%s on %s"
+ % ("delete" if delete else "set",
+ qname[0], qname[1], child))
- loaded = self._loadCache(childNames=(child.fp.basename(),))
- propertyCache.update(loaded.iteritems())
+ def deleteProperty(self, child, qname):
+ return self.setProperty(child, qname, delete=True)
def flushCache(self, child):
path = child.fp.path
@@ -234,22 +265,6 @@
if not result:
raise MemcacheError("Unable to flush cache on %s" % (child,))
- def deleteProperty(self, child, qname):
- propertyCache, key, childCache, token = self.childCache(child)
-
- del childCache[qname]
-
- client = self.memcacheClient()
- if client is not None:
- result = client.set(key, childCache, time=self.cacheTimeout, token=token)
- if not result:
- delattr(self, "_propertyCache")
- raise MemcacheError("Unable to delete property {%s}%s on %s"
- % (qname[0], qname[1], child))
-
- loaded = self._loadCache(childNames=(child.fp.basename(),))
- propertyCache.update(loaded.iteritems())
-
def propertyStoreForChild(self, child, childPropertyStore):
return self.ChildPropertyStore(self, child, childPropertyStore)
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_config.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_config.py 2009-06-17 18:51:12 UTC (rev 4353)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_config.py 2009-06-17 21:52:39 UTC (rev 4354)
@@ -14,12 +14,12 @@
# limitations under the License.
##
-from twisted.trial import unittest
+from plistlib import writePlist
-from twistedcaldav.py.plistlib import writePlist
from twistedcaldav.log import logLevelForNamespace
-from twistedcaldav.config import config, defaultConfig, ConfigurationError
+from twistedcaldav.config import config, defaultConfig
from twistedcaldav.static import CalDAVFile
+from twistedcaldav.test.util import TestCase
testConfig = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@@ -33,7 +33,7 @@
<integer>8008</integer>
<key>DefaultLogLevel</key>
- <string>warn</string>
+ <string>info</string>
<key>LogLevels</key>
<dict>
<key>some.namespace</key>
@@ -45,15 +45,15 @@
"""
def _testResponseCompression(testCase):
- from twistedcaldav.config import config
testCase.assertEquals(config.ResponseCompression, False)
-class ConfigTests(unittest.TestCase):
+class ConfigTests(TestCase):
def setUp(self):
+ TestCase.setUp(self)
config.update(defaultConfig)
self.testConfig = self.mktemp()
- open(self.testConfig, 'w').write(testConfig)
+ open(self.testConfig, "w").write(testConfig)
def tearDown(self):
config.setDefaults(defaultConfig)
@@ -100,7 +100,7 @@
self.assertEquals(config.HTTPPort, 8008)
- config.update({'HTTPPort': 80})
+ config.update({"HTTPPort": 80})
self.assertEquals(config.HTTPPort, 80)
@@ -109,82 +109,81 @@
self.assertEquals(config.HTTPPort, 8008)
def testSetAttr(self):
- self.assertNotIn('BindAddresses', config.__dict__)
+ self.assertNotIn("BindAddresses", config.__dict__)
- config.BindAddresses = ['127.0.0.1']
+ config.BindAddresses = ["127.0.0.1"]
- self.assertNotIn('BindAddresses', config.__dict__)
+ self.assertNotIn("BindAddresses", config.__dict__)
- self.assertEquals(config.BindAddresses, ['127.0.0.1'])
+ self.assertEquals(config.BindAddresses, ["127.0.0.1"])
def testUpdating(self):
self.assertEquals(config.SSLPort, 0)
- config.update({'SSLPort': 8443})
+ config.update({"SSLPort": 8443})
self.assertEquals(config.SSLPort, 8443)
def testMerge(self):
- self.assertEquals(config.MultiProcess["LoadBalancer"]["Enabled"], True)
+ self.assertEquals(config.MultiProcess.LoadBalancer.Enabled, True)
- config.update({'MultiProcess': {}})
+ config.update({"MultiProcess": {}})
- self.assertEquals(config.MultiProcess["LoadBalancer"]["Enabled"], True)
+ self.assertEquals(config.MultiProcess.LoadBalancer.Enabled, True)
def testDirectoryService_noChange(self):
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
config.update({"DirectoryService": {}})
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
def testDirectoryService_sameType(self):
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
config.update({"DirectoryService": {"type": "twistedcaldav.directory.xmlfile.XMLDirectoryService"}})
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
def testDirectoryService_newType(self):
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
config.update({"DirectoryService": {"type": "twistedcaldav.directory.appleopendirectory.OpenDirectoryService"}})
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.appleopendirectory.OpenDirectoryService")
- self.assertNotIn("xmlFile", config.DirectoryService["params"])
- self.assertEquals(config.DirectoryService["params"]["node"], "/Search")
- self.assertEquals(config.DirectoryService["params"]["requireComputerRecord"], True)
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.appleopendirectory.OpenDirectoryService")
+ self.assertNotIn("xmlFile", config.DirectoryService.params)
+ self.assertEquals(config.DirectoryService.params.node, "/Search")
+ self.assertEquals(config.DirectoryService.params.requireComputerRecord, True)
+ self.assertEquals(config.DirectoryService.params.cacheTimeout, 30)
def testDirectoryService_newParam(self):
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
config.update({"DirectoryService": {"type": "twistedcaldav.directory.appleopendirectory.OpenDirectoryService"}})
- config.update({"DirectoryService": {"params": {"requireComputerRecord": False}}})
+ config.update({"DirectoryService": {"params": {
+ "requireComputerRecord": False,
+ "cacheTimeout": 12345,
+ }}})
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.appleopendirectory.OpenDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["node"], "/Search")
- self.assertEquals(config.DirectoryService["params"]["requireComputerRecord"], False)
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.appleopendirectory.OpenDirectoryService")
+ self.assertEquals(config.DirectoryService.params.node, "/Search")
+ self.assertEquals(config.DirectoryService.params.requireComputerRecord, False)
+ self.assertEquals(config.DirectoryService.params.cacheTimeout, 12345)
- def testDirectoryService_badParam(self):
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
-
- self.assertRaises(ConfigurationError, config.update, {"DirectoryService": {"params": {"requireComputerRecord": False}}})
-
def testDirectoryService_unknownType(self):
- self.assertEquals(config.DirectoryService["type"], "twistedcaldav.directory.xmlfile.XMLDirectoryService")
- self.assertEquals(config.DirectoryService["params"]["xmlFile"], "/etc/caldavd/accounts.xml")
+ self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
+ self.assertEquals(config.DirectoryService.params.xmlFile, "/etc/caldavd/accounts.xml")
config.update({"DirectoryService": {"type": "twistedcaldav.test.test_config.SuperDuperAwesomeService"}})
#self.assertEquals(
- # config.DirectoryService["params"],
+ # config.DirectoryService.params,
# SuperDuperAwesomeService.defaultParameters
#)
@@ -195,7 +194,7 @@
config.loadConfig(self.testConfig)
- config.updateDefaults({'SSLPort': 8009})
+ config.updateDefaults({"SSLPort": 8009})
self.assertEquals(config.SSLPort, 8009)
@@ -203,15 +202,15 @@
self.assertEquals(config.SSLPort, 8009)
- config.updateDefaults({'SSLPort': 0})
+ config.updateDefaults({"SSLPort": 0})
def testMergeDefaults(self):
- config.updateDefaults({'MultiProcess': {}})
+ config.updateDefaults({"MultiProcess": {}})
self.assertEquals(config._defaults["MultiProcess"]["LoadBalancer"]["Enabled"], True)
def testSetDefaults(self):
- config.updateDefaults({'SSLPort': 8443})
+ config.updateDefaults({"SSLPort": 8443})
config.setDefaults(defaultConfig)
@@ -220,9 +219,9 @@
self.assertEquals(config.SSLPort, 0)
def testCopiesDefaults(self):
- config.updateDefaults({'Foo': 'bar'})
+ config.updateDefaults({"Foo": "bar"})
- self.assertNotIn('Foo', defaultConfig)
+ self.assertNotIn("Foo", defaultConfig)
def testComplianceClasses(self):
resource = CalDAVFile("/")
@@ -237,16 +236,16 @@
"""
Logging module configures properly.
"""
- self.assertEquals(logLevelForNamespace(None), "info")
- self.assertEquals(logLevelForNamespace("some.namespace"), "info")
+ self.assertEquals(logLevelForNamespace(None), "warn")
+ self.assertEquals(logLevelForNamespace("some.namespace"), "warn")
config.loadConfig(self.testConfig)
- self.assertEquals(logLevelForNamespace(None), "warn")
+ self.assertEquals(logLevelForNamespace(None), "info")
self.assertEquals(logLevelForNamespace("some.namespace"), "debug")
writePlist({}, self.testConfig)
config.reload()
- self.assertEquals(logLevelForNamespace(None), "info")
- self.assertEquals(logLevelForNamespace("some.namespace"), "info")
+ self.assertEquals(logLevelForNamespace(None), "warn")
+ self.assertEquals(logLevelForNamespace("some.namespace"), "warn")
Added: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_memcacheprops.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_memcacheprops.py (rev 0)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_memcacheprops.py 2009-06-17 21:52:39 UTC (rev 4354)
@@ -0,0 +1,192 @@
+##
+# Copyright (c) 2009 Apple Computer, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+##
+
+"""
+Test memcacheprops.
+"""
+
+import os
+
+from twisted.web2.http import HTTPError
+
+from twistedcaldav.config import config
+from twistedcaldav.memcacheprops import MemcachePropertyCollection
+from twistedcaldav.test.util import InMemoryPropertyStore
+from twistedcaldav.test.util import TestCase
+
+
+
+class StubCollection(object):
+
+ def __init__(self, path, childNames):
+ self.path = path
+ self.fp = StubFP(path)
+ self.children = {}
+
+ for childName in childNames:
+ self.children[childName] = StubResource(self, path, childName)
+
+ def listChildren(self):
+ return self.children.iterkeys()
+
+ def getChild(self, childName):
+ return self.children[childName]
+
+ def propertyCollection(self):
+ if not hasattr(self, "_propertyCollection"):
+ self._propertyCollection = MemcachePropertyCollection(self)
+ return self._propertyCollection
+
+
+class StubResource(object):
+
+ def __init__(self, parent, path, name):
+ self.parent = parent
+ self.fp = StubFP(os.path.join(path, name))
+
+ def deadProperties(self):
+ if not hasattr(self, "_dead_properties"):
+ self._dead_properties = self.parent.propertyCollection().propertyStoreForChild(self, InMemoryPropertyStore())
+ return self._dead_properties
+
+class StubFP(object):
+
+ def __init__(self, path):
+ self.path = path
+
+ def child(self, childName):
+ class _Child(object):
+ def __init__(self, path):
+ self.path = path
+ return _Child(os.path.join(self.path, childName))
+
+ def basename(self):
+ return os.path.basename(self.path)
+
+class StubProperty(object):
+
+ def __init__(self, ns, name, value=None):
+ self.ns = ns
+ self.name = name
+ self.value = value
+
+ def qname(self):
+ return self.ns, self.name
+
+
+ def __repr__(self):
+ return "{%s}%s = %s" % (self.ns, self.name, self.value)
+
+
+class MemcachePropertyCollectionTestCase(TestCase):
+ """
+ Test MemcacheProprtyCollection
+ """
+
+ def getColl(self):
+ return StubCollection("calendars", ["a", "b", "c"])
+
+ def test_setget(self):
+
+ child1 = self.getColl().getChild("a")
+ child1.deadProperties().set(StubProperty("ns1:", "prop1", value="val1"))
+
+ child2 = self.getColl().getChild("a")
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop1")).value,
+ "val1")
+
+ child2.deadProperties().set(StubProperty("ns1:", "prop1", value="val2"))
+
+ # force memcache to be consulted (once per collection per request)
+ child1 = self.getColl().getChild("a")
+
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop1")).value,
+ "val2")
+
+ def test_merge(self):
+ child1 = self.getColl().getChild("a")
+ child2 = self.getColl().getChild("a")
+ child1.deadProperties().set(StubProperty("ns1:", "prop1", value="val0"))
+ child1.deadProperties().set(StubProperty("ns1:", "prop2", value="val0"))
+ child1.deadProperties().set(StubProperty("ns1:", "prop3", value="val0"))
+
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop1")).value,
+ "val0")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop2")).value,
+ "val0")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop3")).value,
+ "val0")
+
+ child2.deadProperties().set(StubProperty("ns1:", "prop1", value="val1"))
+ child1.deadProperties().set(StubProperty("ns1:", "prop3", value="val3"))
+
+ # force memcache to be consulted (once per collection per request)
+ child2 = self.getColl().getChild("a")
+
+ # verify properties
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop1")).value,
+ "val1")
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop2")).value,
+ "val0")
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop3")).value,
+ "val3")
+
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop1")).value,
+ "val1")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop2")).value,
+ "val0")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop3")).value,
+ "val3")
+
+ def test_delete(self):
+ child1 = self.getColl().getChild("a")
+ child2 = self.getColl().getChild("a")
+ child1.deadProperties().set(StubProperty("ns1:", "prop1", value="val0"))
+ child1.deadProperties().set(StubProperty("ns1:", "prop2", value="val0"))
+ child1.deadProperties().set(StubProperty("ns1:", "prop3", value="val0"))
+
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop1")).value,
+ "val0")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop2")).value,
+ "val0")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop3")).value,
+ "val0")
+
+ child2.deadProperties().set(StubProperty("ns1:", "prop1", value="val1"))
+ child1.deadProperties().delete(("ns1:", "prop1"))
+ self.assertRaises(HTTPError, child1.deadProperties().get, ("ns1:", "prop1"))
+
+ self.assertFalse(child1.deadProperties().contains(("ns1:", "prop1")))
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop2")).value,
+ "val0")
+ self.assertEquals(child1.deadProperties().get(("ns1:", "prop3")).value,
+ "val0")
+
+ # force memcache to be consulted (once per collection per request)
+ child2 = self.getColl().getChild("a")
+
+ # verify properties
+ self.assertFalse(child2.deadProperties().contains(("ns1:", "prop1")))
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop2")).value,
+ "val0")
+ self.assertEquals(child2.deadProperties().get(("ns1:", "prop3")).value,
+ "val0")
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/util.py 2009-06-17 18:51:12 UTC (rev 4353)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/util.py 2009-06-17 21:52:39 UTC (rev 4354)
@@ -14,7 +14,10 @@
# limitations under the License.
##
+from __future__ import with_statement
+
import os
+import xattr
from twisted.python.failure import Failure
from twisted.internet.defer import succeed, fail
@@ -22,9 +25,12 @@
from twistedcaldav.config import config
from twistedcaldav.static import CalDAVFile
+import memcacheclient
import twisted.web2.dav.test.util
+from twisted.internet.base import DelayedCall
+DelayedCall.debug = True
class TestCase(twisted.web2.dav.test.util.TestCase):
resource_class = CalDAVFile
@@ -35,8 +41,106 @@
dataroot = self.mktemp()
os.mkdir(dataroot)
config.DataRoot = dataroot
+ config.Memcached.ClientEnabled = False
+ config.Memcached.ServerEnabled = False
+ memcacheclient.ClientFactory.allowTestCache = True
+ def createHierarchy(self, structure):
+ root = self.mktemp()
+ os.mkdir(root)
+ def createChildren(parent, subStructure):
+ for childName, childStructure in subStructure.iteritems():
+
+ if childName.startswith("@"):
+ continue
+
+ childPath = os.path.join(parent, childName)
+ if childStructure.has_key("@contents"):
+ # This is a file
+ with open(childPath, "w") as child:
+ child.write(childStructure["@contents"])
+
+ else:
+ # This is a directory
+ os.mkdir(childPath)
+ createChildren(childPath, childStructure)
+
+ if childStructure.has_key("@xattrs"):
+ xattrs = childStructure["@xattrs"]
+ for attr, value in xattrs.iteritems():
+ xattr.setxattr(childPath, attr, value)
+
+ createChildren(root, structure)
+ return root
+
+ def verifyHierarchy(self, root, structure):
+
+ def verifyChildren(parent, subStructure):
+
+ actual = set([child for child in os.listdir(parent)])
+
+ for childName, childStructure in subStructure.iteritems():
+
+ if childName.startswith("@"):
+ continue
+
+ if childName in actual:
+ actual.remove(childName)
+
+ childPath = os.path.join(parent, childName)
+
+ if not os.path.exists(childPath):
+ print "Missing:", childPath
+ return False
+
+ if childStructure.has_key("@contents"):
+ # This is a file
+ if childStructure["@contents"] is None:
+ # We don't care about the contents
+ pass
+ else:
+ with open(childPath) as child:
+ contents = child.read()
+ if contents != childStructure["@contents"]:
+ print "Contents mismatch:", childPath
+ print "Expected:\n%s\n\nActual:\n%s\n" % (childStructure["@contents"], contents)
+ return False
+
+ else:
+ # This is a directory
+ if not verifyChildren(childPath, childStructure):
+ return False
+
+ if childStructure.has_key("@xattrs"):
+ xattrs = childStructure["@xattrs"]
+ for attr, value in xattrs.iteritems():
+ if isinstance(value, str):
+ try:
+ if xattr.getxattr(childPath, attr) != value:
+ print "Xattr mismatch:", childPath, attr
+ print (xattr.getxattr(childPath, attr), " != ", value)
+ return False
+ except:
+ return False
+ else: # method
+ if not value(xattr.getxattr(childPath, attr)):
+ return False
+
+ for attr, value in xattr.xattr(childPath).iteritems():
+ if attr not in xattrs:
+ return False
+
+ if actual:
+ # There are unexpected children
+ print "Unexpected:", actual
+ return False
+
+ return True
+
+ return verifyChildren(root, structure)
+
+
class InMemoryPropertyStore(object):
def __init__(self):
class _FauxPath(object):
@@ -57,8 +161,17 @@
def set(self, property):
self._properties[property.qname()] = property
+ def delete(self, qname):
+ try:
+ del self._properties[qname]
+ except KeyError:
+ pass
+ def list(self):
+ return self._properties.iterkeys()
+
+
class StubCacheChangeNotifier(object):
def __init__(self, *args, **kwargs):
pass
@@ -97,8 +210,6 @@
if key in self._timeouts:
self._timeouts[key].cancel()
- from twisted.internet.base import DelayedCall
- DelayedCall.debug = True
self._timeouts[key] = self._reactor.callLater(
expireTime,
@@ -113,7 +224,7 @@
return succeed(True)
- except Exception, err:
+ except Exception:
return fail(Failure())
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090617/b49a47bf/attachment-0001.html>
More information about the calendarserver-changes
mailing list