[CalendarServer-changes] [12430] CalendarServer/branches/users/cdaboo/pod-migration/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:20:06 PDT 2014
Revision: 12430
http://trac.calendarserver.org//changeset/12430
Author: cdaboo at apple.com
Date: 2014-01-22 19:26:38 -0800 (Wed, 22 Jan 2014)
Log Message:
-----------
Checkpoint: cross-pod serialization of properties.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/base.py
CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/sql.py
CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/conduit.py
CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/test/test_conduit.py
CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/sql.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/memory.py
Modified: CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/base.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/base.py 2014-01-23 02:20:26 UTC (rev 12429)
+++ CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/base.py 2014-01-23 03:26:38 UTC (rev 12430)
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
+from twisted.internet.defer import inlineCallbacks
"""
Base property store.
@@ -254,15 +255,33 @@
return key in self._globalKeys
+ @inlineCallbacks
def copyAllProperties(self, other):
"""
Copy all the properties from another store into this one. This needs to be done
- independently of the UID. Each underlying store will need to implement this.
+ independently of the UID.
"""
- pass
+ other_props = yield other.serialize()
+ yield self.deserialize(other_props)
+ def serialize(self):
+ """
+ Return a C{dict} of all properties, where the dict key is a C{tuple} of the property name
+ and the viewer UID, and the values are C{str} representations of the XML.
+ """
+ raise NotImplementedError()
+
+
+ def deserialize(self, props):
+ """
+ Copy all properties from the specified C{dict} into this store. The dict comes from an L{serialize} call.
+ """
+ raise NotImplementedError()
+
+
+
# FIXME: Actually, we should replace this with calls to IPropertyName()
def validKey(key):
# Used by implementations to verify that keys are valid
Added: CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/memory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/memory.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/memory.py 2014-01-23 03:26:38 UTC (rev 12430)
@@ -0,0 +1,109 @@
+# -*- test-case-name: txdav.base.propertystore.test.test_xattr -*-
+##
+# Copyright (c) 2010-2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from twisted.internet.defer import succeed
+
+"""
+Property store using a dict - read-only (except for initial insert). This is used
+to mirror properties on a resource on another pod.
+"""
+
+from txdav.base.propertystore.base import AbstractPropertyStore, validKey, \
+ PropertyName
+from txdav.idav import PropertyChangeNotAllowedError
+from txdav.xml.parser import WebDAVDocument
+
+
+__all__ = [
+ "PropertyStore",
+]
+
+
+
+class PropertyStore(AbstractPropertyStore):
+ """
+ Property store using a C{dict}. C{dict} keys are a C{tuple} of the property name and user
+ id (both as C{str}. Property values are the C{str} representation of the XML value.
+ """
+
+ def __init__(self, defaultuser, properties):
+ """
+ Initialize a L{PropertyStore}.
+ """
+ super(PropertyStore, self).__init__(defaultuser)
+
+ self.properties = properties
+
+
+ def __str__(self):
+ return "<%s>" % (self.__class__.__name__)
+
+
+ #
+ # Required implementations
+ #
+
+ def _getitem_uid(self, key, uid):
+ validKey(key)
+ try:
+ value = self.properties[(key.toString(), uid)]
+ except KeyError:
+ raise KeyError(key)
+
+ return WebDAVDocument.fromString(value).root_element
+
+
+ def _setitem_uid(self, key, value, uid):
+ validKey(key)
+ raise PropertyChangeNotAllowedError("Property store is read-only.", (key,))
+
+
+ def _delitem_uid(self, key, uid):
+ validKey(key)
+ raise PropertyChangeNotAllowedError("Property store is read-only.", (key,))
+
+
+ def _keys_uid(self, uid):
+ for cachedKey, cachedUID in self.properties.keys():
+ if cachedUID == uid:
+ yield PropertyName.fromString(cachedKey)
+
+
+ def _removeResource(self):
+ pass
+
+
+ def flush(self):
+ return None
+
+
+ def abort(self):
+ return None
+
+
+ def serialize(self):
+ """
+ The dict used in this class is already the serialized format.
+ """
+ return succeed(self.properties)
+
+
+ def deserialize(self, props):
+ """
+ The dict being passed in is already the format we need.
+ """
+ self.properties = props
+ return succeed(None)
Modified: CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/sql.py 2014-01-23 02:20:26 UTC (rev 12429)
+++ CalendarServer/branches/users/cdaboo/pod-migration/txdav/base/propertystore/sql.py 2014-01-23 03:26:38 UTC (rev 12430)
@@ -378,23 +378,43 @@
@inlineCallbacks
- def copyAllProperties(self, other):
+ def serialize(self):
"""
- Copy all the properties from another store into this one. This needs to be done
- independently of the UID.
+ Return a C{dict} of all properties, where the dict key is a C{tuple} of the property name
+ and the viewer UID, and the values are C{str} representations of the XML.
"""
+ results = {}
+ rows = yield self._allWithID.on(self._txn, resourceID=self._resourceID)
+ for key_str, uid, value_str in rows:
+ results[(key_str, uid)] = value_str
+ returnValue(results)
- rows = yield other._allWithID.on(other._txn, resourceID=other._resourceID)
- for key_str, uid, value_str in rows:
- wasCached = [(key_str, uid) in self._cached]
- if wasCached[0]:
- yield self._updateQuery.on(
- self._txn, resourceID=self._resourceID, value=value_str,
- name=key_str, uid=uid)
+
+ @inlineCallbacks
+ def deserialize(self, props):
+ """
+ Copy all properties from the specified C{dict} into this store. The dict comes from an L{serialize} call.
+ """
+
+ for key, value_str in props.items():
+ key_str, uid = key
+ if key in self._cached:
+ if self._cached[key] != value_str:
+ yield self._updateQuery.on(
+ self._txn,
+ resourceID=self._resourceID,
+ value=value_str,
+ name=key_str,
+ uid=uid
+ )
else:
yield self._insertQuery.on(
- self._txn, resourceID=self._resourceID, value=value_str,
- name=key_str, uid=uid)
+ self._txn,
+ resourceID=self._resourceID,
+ value=value_str,
+ name=key_str,
+ uid=uid
+ )
# Invalidate entire set of cached per-user data for this resource and reload
self._cached = {}
Modified: CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/conduit.py 2014-01-23 02:20:26 UTC (rev 12429)
+++ CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/conduit.py 2014-01-23 03:26:38 UTC (rev 12430)
@@ -16,7 +16,7 @@
from twext.python.log import Logger
-from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
from twisted.python.reflect import namedClass
from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
@@ -699,9 +699,11 @@
"message": str(e),
})
+ if transform is not None:
+ value = yield maybeDeferred(transform, value)
returnValue({
"result": "ok",
- "value": transform(value) if transform is not None else value,
+ "value": value,
})
@@ -794,8 +796,15 @@
@staticmethod
+ @inlineCallbacks
def _to_externalize_list(value):
- return [v.externalize() for v in value] if value is not None else None
+ results = None
+ if value is not None:
+ results = []
+ for v in value:
+ v = yield v.externalize()
+ results.append(v)
+ returnValue(results)
@classmethod
@@ -970,7 +979,11 @@
action["keywords"] = kwargs
result = yield self.sendRequest(target.transaction(), recipient, action)
if result["result"] == "ok":
- returnValue(result["value"] if transform is None else transform(result["value"]))
+ if transform is not None:
+ value = yield maybeDeferred(transform, result["value"])
+ else:
+ value = result["value"]
+ returnValue(value)
elif result["result"] == "exception":
raise namedClass(result["class"])(result["message"])
@@ -1005,9 +1018,11 @@
"message": str(e),
})
+ if transform is not None:
+ value = yield maybeDeferred(transform, value)
returnValue({
"result": "ok",
- "value": transform(value) if transform is not None else value,
+ "value": value,
})
Modified: CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/test/test_conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2014-01-23 02:20:26 UTC (rev 12429)
+++ CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2014-01-23 03:26:38 UTC (rev 12430)
@@ -1103,7 +1103,7 @@
remote_home = yield self.homeUnderTest(name="puser01", create=True)
remote_home._migration = _MIGRATION_STATUS_MIGRATING
- result = yield remote_home.get()
+ result = yield remote_home.getExternal()
self.assertTrue(isinstance(result, CalendarHomeExternal))
self.assertEqual(result.name(), remote_home.name())
Modified: CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/sql.py 2014-01-23 02:20:26 UTC (rev 12429)
+++ CalendarServer/branches/users/cdaboo/pod-migration/txdav/common/datastore/sql.py 2014-01-23 03:26:38 UTC (rev 12430)
@@ -55,6 +55,7 @@
from txdav.base.datastore.util import QueryCacher
from txdav.base.datastore.util import normalizeUUIDOrNot
from txdav.base.propertystore.base import PropertyName
+from txdav.base.propertystore.memory import PropertyStore as MemoryPropertyStore
from txdav.base.propertystore.none import PropertyStore as NonePropertyStore
from txdav.base.propertystore.sql import PropertyStore
from txdav.caldav.icalendarstore import ICalendarTransaction, ICalendarStore
@@ -1640,7 +1641,7 @@
@classmethod
@inlineCallbacks
- def makeClass(cls, txn, homeData, metadataData):
+ def makeClass(cls, txn, homeData, metadataData, properties=None):
"""
Build the actual home class taking into account the possibility that we might need to
switch in the external version of the class.
@@ -1673,7 +1674,10 @@
for attr, value in zip(cls.metadataAttributes(), metadataData):
setattr(home, attr, value)
- yield home._loadPropertyStore()
+ if properties is None:
+ yield home._loadPropertyStore()
+ else:
+ home._propertyStore = MemoryPropertyStore(home.uid(), properties)
for factory_type, factory in txn._notifierFactories.items():
home.addNotifier(factory_type, factory.newNotifier(home))
@@ -1770,6 +1774,7 @@
return succeed(None)
+ @inlineCallbacks
def externalize(self):
"""
Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
@@ -1779,7 +1784,8 @@
serialized = {}
serialized["home"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.homeAttributes()])
serialized["metadata"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.metadataAttributes()])
- return serialized
+ serialized["properties"] = (yield self.properties().serialize())
+ returnValue(serialized)
@classmethod
@@ -1794,7 +1800,7 @@
home = [mapping["home"].get(row[1:]) for row in cls.homeAttributes()]
metadata = [mapping["metadata"].get(row[1:]) for row in cls.metadataAttributes()]
- child = yield cls.makeClass(txn, home, metadata)
+ child = yield cls.makeClass(txn, home, metadata, properties=mapping["properties"])
returnValue(child)
@@ -4217,7 +4223,7 @@
@classmethod
@inlineCallbacks
- def makeClass(cls, home, bindData, additionalBindData, metadataData, propstore=None, ownerHome=None):
+ def makeClass(cls, home, bindData, additionalBindData, metadataData, propstore=None, ownerHome=None, properties=None):
"""
Given the various database rows, build the actual class.
@@ -4281,11 +4287,14 @@
for attr, value in zip(child.metadataAttributes(), metadataData):
setattr(child, attr, value)
- # We have to re-adjust the property store object to account for possible shared
- # collections as previously we loaded them all as if they were owned
- if propstore and bindMode != _BIND_MODE_OWN:
- propstore._setDefaultUserUID(ownerHome.uid())
- yield child._loadPropertyStore(propstore)
+ if properties is None:
+ # We have to re-adjust the property store object to account for possible shared
+ # collections as previously we loaded them all as if they were owned
+ if propstore and bindMode != _BIND_MODE_OWN:
+ propstore._setDefaultUserUID(ownerHome.uid())
+ yield child._loadPropertyStore(propstore)
+ else:
+ home._propertyStore = MemoryPropertyStore(ownerHome.uid(), properties)
returnValue(child)
@@ -4565,6 +4574,7 @@
returnValue(child)
+ @inlineCallbacks
def externalize(self):
"""
Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
@@ -4575,7 +4585,8 @@
serialized["bind"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.bindAttributes()])
serialized["additionalBind"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.additionalBindAttributes()])
serialized["metadata"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.metadataAttributes()])
- return serialized
+ serialized["properties"] = (yield self.properties().serialize())
+ returnValue(serialized)
@classmethod
@@ -4591,7 +4602,7 @@
bind = [mapping["bind"].get(row[1:]) for row in cls.bindAttributes()]
additionalBind = [mapping["additionalBind"].get(row[1:]) for row in cls.additionalBindAttributes()]
metadata = [mapping["metadata"].get(row[1:]) for row in cls.metadataAttributes()]
- child = yield cls.makeClass(parent, bind, additionalBind, metadata)
+ child = yield cls.makeClass(parent, bind, additionalBind, metadata, properties=mapping["properties"])
returnValue(child)
@@ -5848,7 +5859,7 @@
and reconstituted at the other end. Note that the other end may have a different schema so
the attributes may not match exactly and will need to be processed accordingly.
"""
- return dict([(attr[1:], getattr(self, attr, None)) for attr in itertools.chain(self._rowAttributes(), self._otherSerializedAttributes())])
+ return succeed(dict([(attr[1:], getattr(self, attr, None)) for attr in itertools.chain(self._rowAttributes(), self._otherSerializedAttributes())]))
@classmethod
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/7cd7bf89/attachment.html>
More information about the calendarserver-changes
mailing list