[CalendarServer-changes] [11255] CalendarServer/branches/users/cdaboo/group-sync/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Wed May 29 12:02:14 PDT 2013
Revision: 11255
http://trac.calendarserver.org//changeset/11255
Author: cdaboo at apple.com
Date: 2013-05-29 12:02:14 -0700 (Wed, 29 May 2013)
Log Message:
-----------
Checkpoint for shared group sync work.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/group-sync/txdav/carddav/datastore/sql.py
CalendarServer/branches/users/cdaboo/group-sync/txdav/common/datastore/sql.py
Modified: CalendarServer/branches/users/cdaboo/group-sync/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/group-sync/txdav/carddav/datastore/sql.py 2013-05-29 18:59:54 UTC (rev 11254)
+++ CalendarServer/branches/users/cdaboo/group-sync/txdav/carddav/datastore/sql.py 2013-05-29 19:02:14 UTC (rev 11255)
@@ -28,7 +28,7 @@
from uuid import uuid4
from twext.enterprise.dal.syntax import Delete, Insert, Len, Parameter, \
- Update, Union, Max, Select, utcNowSQL
+ Update, Max, Select, utcNowSQL
from twext.enterprise.locking import NamedLock
from twext.python.clsprop import classproperty
from twext.web2.http import HTTPError
@@ -269,44 +269,29 @@
DAL Select statement to find the sync token.
"""
rev = cls._revisionsSchema
- bind = cls._bindSchema
return Select(
[Max(rev.REVISION)],
- # active shared address books
- From=Select(
- [rev.REVISION],
- From=rev,
- Where=(
- rev.RESOURCE_ID.In(
- Select(
- [bind.RESOURCE_ID],
- From=bind,
- Where=bind.HOME_RESOURCE_ID == Parameter("resourceID"),
- )
- )
- ),
- SetExpression=Union(
- # deleted shared address books
- Select(
- [rev.REVISION],
- From=rev,
- Where=(rev.HOME_RESOURCE_ID == Parameter("resourceID")).And(rev.RESOURCE_ID == None),
- SetExpression=Union(
- # owned address book: owned address book cannot be deleted: See AddressBook.remove()
- Select(
- [rev.REVISION],
- From=rev,
- Where=(rev.HOME_RESOURCE_ID == Parameter("resourceID")).And(rev.RESOURCE_ID == rev.HOME_RESOURCE_ID),
- ),
- optype=Union.OPTYPE_ALL,
- )
- ),
- optype=Union.OPTYPE_ALL,
- )
- ),
+ # Any changes related to this address book home. These will be changes in the main
+ # address book as well as add/remove of shared address book collections (but NOT
+ # changes to resources in shared collections)
+ From=rev,
+ Where=rev.HOME_RESOURCE_ID == Parameter("resourceID"),
)
+ @inlineCallbacks
+ def syncToken(self):
+ """
+ For the main addressbook and shared address books we can do a single query to find the max revision.
+ For group address books we have to iterate each one to get their max revision.
+ Then we return the max of all the results.
+ """
+ if self._syncTokenRevision is None:
+ self._syncTokenRevision = (yield self._syncTokenQuery.on(
+ self._txn, resourceID=self._resourceID))[0][0]
+ returnValue("%s_%s" % (self._resourceID, self._syncTokenRevision))
+
+
@classproperty
def _changesQuery(cls): #@NoSelf
rev = cls._revisionsSchema
@@ -470,9 +455,6 @@
yield self.unshare() # storebridge should already have done this
- # Note that revision table is NOT queried for removes
- yield self._updateRevision(self.name())
-
yield self.properties()._removeResource()
yield self._loadPropertyStore()
@@ -941,18 +923,29 @@
@return: an iterable of C{str}s.
"""
+
+ # The main address book
names = set([home.addressbook().name()])
+ # The shared address books
rows = yield cls._acceptedBindForHomeID.on(
home._txn, homeID=home._resourceID
)
- rows.extend((yield AddressBookObject._acceptedBindForHomeID.on(
- home._txn, homeID=home._resourceID
- )))
for row in rows:
bindMode, homeID, resourceID, bindName, bindStatus, bindRevision, bindMessage = row[:cls.bindColumnCount] #@UnusedVariable
ownerHome = yield home._txn.homeWithResourceID(home._homeType, resourceID)
names |= set([ownerHome.shareeAddressBookName()])
+
+ # The group address books
+ rows = (yield AddressBookObject._acceptedBindForHomeID.on(
+ home._txn, homeID=home._resourceID
+ ))
+ for row in rows:
+ bindMode, homeID, resourceID, bindName, bindStatus, bindRevision, bindMessage = row[:cls.bindColumnCount] #@UnusedVariable
+ ownerAddressBookID = yield AddressBookObject.ownerAddressBookFromGroupID(home._txn, resourceID)
+ ownerHome = yield home.ownerHomeWithChildID(ownerAddressBookID)
+ names |= set([ownerHome.shareeAddressBookName()])
+
returnValue(tuple(names))
@@ -1289,7 +1282,33 @@
returnValue(deletedBindName)
+ @inlineCallbacks
+ def __syncToken(self):
+ if self.owned() or self.fullyShared():
+ result = yield super(AddressBook, self).syncToken()
+ returnValue(result)
+ elif self._syncTokenRevision is None:
+
+ # Get the names of all object resources for all accepted shared groups
+ objectNames = (yield self.listObjectResources())
+
+ rev = self._revisionsSchema
+ self._syncTokenRevision = (yield self.Select(
+ [Max(rev.REVISION)],
+ From=rev,
+ Where=(rev.RESOURCE_ID == Parameter("resourceID")).And
+ (rev.RESOURCE_NAME.In(Parameter("objectNames", len(objectNames)))),
+ ).on(
+ self._txn,
+ resourceID=self.ownerHome()._resourceID,
+ objectNames=objectNames,
+ ))[0][0]
+
+ returnValue(("%s_%s" % (self._resourceID, self._syncTokenRevision,)))
+
+
+
class AddressBookObject(CommonObjectResource, SharingMixIn):
implements(IAddressBookObject)
@@ -1886,6 +1905,7 @@
abo = schema.ADDRESSBOOK_OBJECT
aboForeignMembers = schema.ABO_FOREIGN_MEMBERS
aboMembers = schema.ABO_MEMBERS
+ #groupRevisions = schema.GROUP_REVISIONS
if inserting:
@@ -1945,6 +1965,12 @@
memberIDsToAdd = set(memberIDs) - set(currentMemberIDs)
if memberIDsToDelete:
+# for memberIDToDelete in memberIDsToDelete:
+# yield Update(
+# {groupRevisions.DELETED: True, },
+# Where=(groupRevisions.GROUP_ID == self._resourceID).And
+# (groupRevisions.MEMBER_NAME == memberIDToDelete)
+# ).on(self._txn)
yield self._deleteMembersWithGroupIDAndMemberIDsQuery(self._resourceID, memberIDsToDelete).on(
self._txn, memberIDs=memberIDsToDelete
)
@@ -1955,6 +1981,11 @@
aboMembers.ADDRESSBOOK_ID: self._ownerAddressBookResourceID,
aboMembers.MEMBER_ID: memberIDToAdd, }
).on(self._txn)
+# yield Insert(
+# {groupRevisions.GROUP_ID: self._resourceID,
+# groupRevisions.MEMBER_NAME: memberIDToAdd,
+# groupRevisions.DELETED: False, }
+# ).on(self._txn)
# don't bother with aboForeignMembers on address books
if self._resourceID != self._ownerAddressBookResourceID:
Modified: CalendarServer/branches/users/cdaboo/group-sync/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/group-sync/txdav/common/datastore/sql.py 2013-05-29 18:59:54 UTC (rev 11254)
+++ CalendarServer/branches/users/cdaboo/group-sync/txdav/common/datastore/sql.py 2013-05-29 19:02:14 UTC (rev 11255)
@@ -1855,8 +1855,7 @@
@inlineCallbacks
def doChangesQuery(self, revision):
"""
- Do the changes query.
- Subclasses may override.
+ Do the changes query. Subclasses may override.
"""
result = yield self._changesQuery.on(self._txn,
resourceID=self._resourceID,
@@ -1892,6 +1891,8 @@
record a revision for the sharee home and sharee collection name with the "deleted" flag set. That way
the shared collection can be reported as removed.
+ See docstr for L{_SharedSyncLogic} for more details.
+
@param revision: the sync revision to compare to
@type revision: C{str}
@param depth: depth for determine what changed
@@ -2275,6 +2276,71 @@
shared collections.
"""
+ """
+ Here is a detailed description of how we track changes to allow for efficient client synchronization.
+
+ One important aspect of this is to not track every single change as that does not scale well. Instead
+ we only track the last change to any particular resource, so we scale with the number of resources
+ and not the number of changes - the former being bounded, whereas the later is not).
+
+ Another tricky part is the need to record deleted resources - those will not have a corresponding row
+ (and thus a resource_id) anywhere else, so we need to record those by name (which is the quantity that
+ ultimately needs to be reported back to the caller).
+
+ Also, we have to deal with sharing - where changes need to be reported differently for the owner and
+ sharees (e.g., if one sharee is removed from a shared resource they need to be told of the removal, but
+ no one else needs to know). For shared collections, changes to resources in the collection are tracked
+ via the owner's home collection id. The actual share and unshare of the collection are tracked via
+ an entry under the sharee's home collection id.
+
+ Each CommonHome/CommonHomeChild/CommonObject set of resources will have a corresponding XXX_REVISIONS
+ table associated with it. A REVISION_SEQ sequence is used to incrementally assign a number to each
+ change.
+
+ The XXX_REVISIONS table has the following columns:
+
+ XXX_HOME_RESOURCE_ID - the resource_id of the CommonHomeChild to which the set of changes are relevant.
+ For changes to CommonObject's this will be the owner's CommonHome resource_id. For changes to
+ owned CommonHomeChild (create or delete) collections, this will also be the owner's CommonHome
+ resource_id. For changes to shared CommonHomeChild (share accept or share remove) collections this
+ will be the sharee's CommonHome resource_id.
+
+ XXX_RESOURCE_ID - the resource_id of the CommonHomeChild impacted by a change. This is used for
+ CommonHomeChild's that exist. It is null when recording the removal of a CommonHomeChild (in
+ which case the XXX_NAME column will contain the name of the CommonHomeChild when it was
+ removed).
+
+ XXX_NAME - the name of an owned CommonHomeChild when it was removed, or a shared CommonHomeChild
+ when it was accepted or removed.
+
+ RESOURCE_NAME - when recording a change to a CommonObject, this the the name of the object.
+
+ REVISION - the revision number associated with the latest change to the object.
+
+ DELETED - indicates whether the revision is due to a change or deletion of the associated object.
+
+ The possible changes that can occur are as follows:
+
+ 1. New owned/shared collection: XXX_NAME, RESOURCE_NAME will be null, DELETED will be 0.
+ 2. Changed collection: same as 1.
+ 3. Deleted/removed owned/shared collection: XXX_RESOURCE_ID, RESOURCE_NAME will be null, DELETED will be 1.
+
+ 4. New/changed resource: XXX_NAME will be null, DELETED will be 0.
+ 5. Deleted resource: XXX_NAME will be null, DELETED will be 1.
+
+ When determining changes for a CommonHomeChild, all that needs to be done is a query on the resource_id
+ for that collection. That is true for both owned and shared CommonHomeChild collections.
+
+ When determining changes for a CommonHome, owned collections/resources and shared collection (but not
+ shared resource) changes are simply found by a query on the CommonHome resource_id. To find changes to
+ shared resources in existing shared collections, a separate CommonHomeChild query has to be done for
+ each, taking into account the revision where the shared collection first appeared.
+
+ Sync-tokens for CommonHome and CommonHomeChild are simply determined by querying for the maximum revision
+ for all collections/resources within the resource being queried. The token is then formed from the resource_id
+ of the targeted resource and the max revision.
+ """
+
@classproperty
def _childSyncTokenQuery(cls): #@NoSelf
"""
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130529/5c335f46/attachment-0001.html>
More information about the calendarserver-changes
mailing list