[CalendarServer-changes] [14417] CalendarServer/branches/users/cdaboo/pod2pod-migration
source_changes at macosforge.org
source_changes at macosforge.org
Mon Feb 16 12:56:21 PST 2015
Revision: 14417
http://trac.calendarserver.org//changeset/14417
Author: cdaboo at apple.com
Date: 2015-02-16 12:56:21 -0800 (Mon, 16 Feb 2015)
Log Message:
-----------
Checkpoint: re-factor groups/delegates store API to use DAL Record objects to make it easier to serialize/deserialize them for pod2pod.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/principals.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/webadmin/work.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/test/test_work.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/work.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/delegates.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_delegates.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_groups.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_directory.py
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/principals.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/principals.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/principals.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -775,15 +775,11 @@
groupUIDs.append(record.uid)
for groupUID in groupUIDs:
- (
- groupID, name, _ignore_membershipHash, modified, _ignore_extant
- ) = yield txn.groupByUID(
- groupUID
- )
- print("Group: \"{name}\" ({uid})".format(name=name, uid=groupUID))
+ group = yield txn.groupByUID(groupUID)
+ print("Group: \"{name}\" ({uid})".format(name=group.name, uid=group.groupUID))
for txt, readWrite in (("read-only", False), ("read-write", True)):
- delegatorUIDs = yield txn.delegatorsToGroup(groupID, readWrite)
+ delegatorUIDs = yield txn.delegatorsToGroup(group.groupID, readWrite)
for delegatorUID in delegatorUIDs:
delegator = yield directory.recordWithUID(delegatorUID)
print(
@@ -793,12 +789,12 @@
)
print("Group members:")
- memberUIDs = yield txn.groupMemberUIDs(groupID)
+ memberUIDs = yield txn.groupMemberUIDs(group.groupID)
for memberUID in memberUIDs:
record = yield directory.recordWithUID(memberUID)
print(prettyRecord(record))
- print("Last cached: {} GMT".format(modified))
+ print("Last cached: {} GMT".format(group.modified))
print()
yield txn.commit()
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/webadmin/work.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/webadmin/work.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/webadmin/work.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -157,13 +157,13 @@
if workType == PushNotificationWork:
attrs += ("pushID", "priority")
elif workType == ScheduleOrganizerWork:
- attrs += ("icalendarUid", "attendeeCount")
+ attrs += ("icalendarUID", "attendeeCount")
elif workType == ScheduleRefreshWork:
- attrs += ("icalendarUid", "attendeeCount")
+ attrs += ("icalendarUID", "attendeeCount")
elif workType == ScheduleReplyWork:
- attrs += ("icalendarUid",)
+ attrs += ("icalendarUID",)
elif workType == ScheduleAutoReplyWork:
- attrs += ("icalendarUid",)
+ attrs += ("icalendarUID",)
elif workType == GroupCacherPollingWork:
attrs += ()
elif workType == IMIPPollingWork:
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt 2015-02-16 20:56:21 UTC (rev 14417)
@@ -36,7 +36,7 @@
#pyOpenSSL
pycrypto==2.6.1
- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14404#egg=twextpy
+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/branches/users/cdaboo/pod2pod-migration@14416#egg=twextpy
cffi==0.8.6
pycparser==2.10
#twisted
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/test/test_work.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/test/test_work.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/test/test_work.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -250,7 +250,7 @@
work = yield jobs[0].workItem()
self.assertTrue(isinstance(work, ScheduleOrganizerWork))
- self.assertEqual(work.icalendarUid, "12345-67890")
+ self.assertEqual(work.icalendarUID, "12345-67890")
self.assertEqual(scheduleActionFromSQL[work.scheduleAction], "create")
yield work.delete()
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/work.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/work.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/work.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -77,7 +77,7 @@
baseargs = {
"jobID": kwargs.pop("jobID"),
- "icalendarUid": kwargs.pop("icalendarUid"),
+ "icalendarUID": kwargs.pop("icalendarUID"),
"workType": cls.workType()
}
@@ -121,7 +121,7 @@
# cause deadlocks if done in the wrong order
# Row level lock on this item
- locked = yield self.baseWork.trylock(ScheduleWork.icalendarUid == self.icalendarUid)
+ locked = yield self.baseWork.trylock(ScheduleWork.icalendarUID == self.icalendarUID)
if locked:
yield self.trylock()
returnValue(locked)
@@ -136,7 +136,7 @@
"""
self.__dict__["baseWork"] = baseWork
self.__dict__["jobID"] = baseWork.jobID
- self.__dict__["icalendarUid"] = baseWork.icalendarUid
+ self.__dict__["icalendarUID"] = baseWork.icalendarUID
def delete(self):
@@ -174,7 +174,7 @@
if self.workType() == ScheduleOrganizerSendWork.workType():
all = yield self.baseWork.query(
self.transaction,
- (ScheduleWork.icalendarUid == self.icalendarUid).And(ScheduleWork.workID != self.workID),
+ (ScheduleWork.icalendarUID == self.icalendarUID).And(ScheduleWork.workID != self.workID),
order=ScheduleWork.workID,
limit=1,
)
@@ -183,7 +183,7 @@
if work.workType == self.workType():
job = yield JobItem.load(self.transaction, work.jobID)
yield job.update(notBefore=datetime.datetime.utcnow())
- log.debug("ScheduleOrganizerSendWork - promoted job: {id}, UID: '{uid}'", id=work.workID, uid=self.icalendarUid)
+ log.debug("ScheduleOrganizerSendWork - promoted job: {id}, UID: '{uid}'", id=work.workID, uid=self.icalendarUID)
@classmethod
@@ -323,7 +323,7 @@
proposal = (yield txn.enqueue(
cls,
notBefore=notBefore,
- icalendarUid=uid,
+ icalendarUID=uid,
scheduleAction=scheduleActionToSQL[action],
homeResourceID=home.id(),
resourceID=resource.id() if resource else None,
@@ -347,10 +347,10 @@
calendar_old = Component.fromString(self.icalendarTextOld) if self.icalendarTextOld else None
calendar_new = Component.fromString(self.icalendarTextNew) if self.icalendarTextNew else None
- log.debug("ScheduleOrganizerWork - running for ID: {id}, UID: {uid}, organizer: {org}", id=self.workID, uid=self.icalendarUid, org=organizer)
+ log.debug("ScheduleOrganizerWork - running for ID: {id}, UID: {uid}, organizer: {org}", id=self.workID, uid=self.icalendarUID, org=organizer)
# We need to get the UID lock for implicit processing.
- yield NamedLock.acquire(self.transaction, "ImplicitUIDLock:%s" % (hashlib.md5(self.icalendarUid).hexdigest(),))
+ yield NamedLock.acquire(self.transaction, "ImplicitUIDLock:%s" % (hashlib.md5(self.icalendarUID).hexdigest(),))
from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
scheduler = ImplicitScheduler()
@@ -359,7 +359,7 @@
scheduleActionFromSQL[self.scheduleAction],
home,
resource,
- self.icalendarUid,
+ self.icalendarUID,
calendar_old,
calendar_new,
self.smartMerge
@@ -368,15 +368,15 @@
self._dequeued()
except Exception, e:
- log.debug("ScheduleOrganizerWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=self.icalendarUid, err=str(e))
+ log.debug("ScheduleOrganizerWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=self.icalendarUID, err=str(e))
log.debug(traceback.format_exc())
raise
except:
- log.debug("ScheduleOrganizerWork - bare exception ID: {id}, UID: '{uid}'", id=self.workID, uid=self.icalendarUid)
+ log.debug("ScheduleOrganizerWork - bare exception ID: {id}, UID: '{uid}'", id=self.workID, uid=self.icalendarUID)
log.debug(traceback.format_exc())
raise
- log.debug("ScheduleOrganizerWork - done for ID: {id}, UID: {uid}, organizer: {org}", id=self.workID, uid=self.icalendarUid, org=organizer)
+ log.debug("ScheduleOrganizerWork - done for ID: {id}, UID: {uid}, organizer: {org}", id=self.workID, uid=self.icalendarUID, org=organizer)
@@ -418,7 +418,7 @@
proposal = (yield txn.enqueue(
cls,
notBefore=notBefore,
- icalendarUid=uid,
+ icalendarUID=uid,
scheduleAction=scheduleActionToSQL[action],
homeResourceID=home.id(),
resourceID=resource.id() if resource else None,
@@ -449,13 +449,13 @@
log.debug(
"ScheduleOrganizerSendWork - running for ID: {id}, UID: {uid}, organizer: {org}, attendee: {att}",
id=self.workID,
- uid=self.icalendarUid,
+ uid=self.icalendarUID,
org=organizer,
att=self.attendee
)
# We need to get the UID lock for implicit processing.
- yield NamedLock.acquire(self.transaction, "ImplicitUIDLock:%s" % (hashlib.md5(self.icalendarUid).hexdigest(),))
+ yield NamedLock.acquire(self.transaction, "ImplicitUIDLock:%s" % (hashlib.md5(self.icalendarUID).hexdigest(),))
from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
scheduler = ImplicitScheduler()
@@ -464,7 +464,7 @@
scheduleActionFromSQL[self.scheduleAction],
home,
resource,
- self.icalendarUid,
+ self.icalendarUID,
organizer,
self.attendee,
itipmsg,
@@ -486,18 +486,18 @@
self._dequeued()
except Exception, e:
- log.debug("ScheduleOrganizerSendWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=self.icalendarUid, err=str(e))
+ log.debug("ScheduleOrganizerSendWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=self.icalendarUID, err=str(e))
log.debug(traceback.format_exc())
raise
except:
- log.debug("ScheduleOrganizerSendWork - bare exception ID: {id}, UID: '{uid}'", id=self.workID, uid=self.icalendarUid)
+ log.debug("ScheduleOrganizerSendWork - bare exception ID: {id}, UID: '{uid}'", id=self.workID, uid=self.icalendarUID)
log.debug(traceback.format_exc())
raise
log.debug(
"ScheduleOrganizerSendWork - for ID: {id}, UID: {uid}, organizer: {org}, attendee: {att}",
id=self.workID,
- uid=self.icalendarUid,
+ uid=self.icalendarUID,
org=organizer,
att=self.attendee
)
@@ -521,7 +521,7 @@
proposal = (yield txn.enqueue(
cls,
notBefore=notBefore,
- icalendarUid=uid,
+ icalendarUID=uid,
homeResourceID=home.id(),
resourceID=resource.id() if resource else None,
itipMsg=itipmsg.getTextWithTimezones(includeTimezones=not config.EnableTimezonesByReference),
@@ -649,7 +649,7 @@
notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=config.Scheduling.Options.WorkQueues.AttendeeRefreshBatchDelaySeconds)
proposal = (yield txn.enqueue(
cls,
- icalendarUid=organizer_resource.uid(),
+ icalendarUID=organizer_resource.uid(),
homeResourceID=organizer_resource._home.id(),
resourceID=organizer_resource.id(),
attendeeCount=len(attendees),
@@ -676,7 +676,7 @@
log.debug("Schedule refresh for resource-id: {rid} - ignored", rid=self.resourceID)
returnValue(None)
- log.debug("ScheduleRefreshWork - running for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUid)
+ log.debug("ScheduleRefreshWork - running for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUID)
# Get the unique list of pending attendees and split into batch to process
# TODO: do a DELETE ... and rownum <= N returning attendee - but have to fix Oracle to
@@ -707,7 +707,7 @@
notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=config.Scheduling.Options.WorkQueues.AttendeeRefreshBatchIntervalSeconds)
yield self.transaction.enqueue(
self.__class__,
- icalendarUid=self.icalendarUid,
+ icalendarUID=self.icalendarUID,
homeResourceID=self.homeResourceID,
resourceID=self.resourceID,
attendeeCount=len(pendingAttendees),
@@ -721,7 +721,7 @@
self._dequeued()
- log.debug("ScheduleRefreshWork - done for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUid)
+ log.debug("ScheduleRefreshWork - done for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUID)
@inlineCallbacks
@@ -790,7 +790,7 @@
notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=config.Scheduling.Options.WorkQueues.AutoReplyDelaySeconds)
proposal = (yield txn.enqueue(
cls,
- icalendarUid=resource.uid(),
+ icalendarUID=resource.uid(),
homeResourceID=resource._home.id(),
resourceID=resource.id(),
partstat=partstat,
@@ -803,7 +803,7 @@
@inlineCallbacks
def doWork(self):
- log.debug("ScheduleAutoReplyWork - running for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUid)
+ log.debug("ScheduleAutoReplyWork - running for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUID)
# Delete all other work items with the same pushID
yield Delete(
@@ -816,7 +816,7 @@
self._dequeued()
- log.debug("ScheduleAutoReplyWork - done for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUid)
+ log.debug("ScheduleAutoReplyWork - done for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUID)
@inlineCallbacks
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -1878,8 +1878,8 @@
# First check that the actual group membership has changed
if (yield self.updateShareeGroupLink(groupUID)):
- groupID = (yield self._txn.groupByUID(groupUID))[0]
- memberUIDs = yield self._txn.groupMemberUIDs(groupID)
+ group = yield self._txn.groupByUID(groupUID)
+ memberUIDs = yield self._txn.groupMemberUIDs(group.groupID)
boundUIDs = set()
home = self._homeSchema
@@ -2029,39 +2029,36 @@
update schema.GROUP_SHAREE
"""
changed = False
- (
- groupID, _ignore_name, membershipHash, _ignore_modDate,
- _ignore_extant
- ) = yield self._txn.groupByUID(groupUID)
+ group = yield self._txn.groupByUID(groupUID)
gs = schema.GROUP_SHAREE
rows = yield Select(
[gs.MEMBERSHIP_HASH, gs.GROUP_BIND_MODE],
From=gs,
Where=(gs.CALENDAR_ID == self._resourceID).And(
- gs.GROUP_ID == groupID)
+ gs.GROUP_ID == group.groupID)
).on(self._txn)
if rows:
[[gsMembershipHash, gsMode]] = rows
updateMap = {}
- if gsMembershipHash != membershipHash:
- updateMap[gs.MEMBERSHIP_HASH] = membershipHash
+ if gsMembershipHash != group.membershipHash:
+ updateMap[gs.MEMBERSHIP_HASH] = group.membershipHash
if mode is not None and gsMode != mode:
updateMap[gs.GROUP_BIND_MODE] = mode
if updateMap:
yield Update(
updateMap,
Where=(gs.CALENDAR_ID == self._resourceID).And(
- gs.GROUP_ID == groupID
+ gs.GROUP_ID == group.groupID
)
).on(self._txn)
changed = True
else:
yield Insert({
- gs.MEMBERSHIP_HASH: membershipHash,
+ gs.MEMBERSHIP_HASH: group.membershipHash,
gs.GROUP_BIND_MODE: mode,
gs.CALENDAR_ID: self._resourceID,
- gs.GROUP_ID: groupID,
+ gs.GROUP_ID: group.groupID,
}).on(self._txn)
changed = True
@@ -2148,8 +2145,8 @@
# invite every member of group
shareeViews = []
- groupID = (yield self._txn.groupByUID(shareeUID))[0]
- memberUIDs = yield self._txn.groupMemberUIDs(groupID)
+ group = yield self._txn.groupByUID(shareeUID)
+ memberUIDs = yield self._txn.groupMemberUIDs(group.groupID)
for memberUID in memberUIDs:
if memberUID != self._home.uid():
shareeView = yield self.shareeView(memberUID)
@@ -2496,9 +2493,9 @@
groupRecord = yield self.directoryService().recordWithCalendarUserAddress(groupCUA)
if groupRecord:
# get members
- groupID = (yield self._txn.groupByUID(groupRecord.uid))[0]
- if groupID is not None:
- members = yield self._txn.groupMembers(groupID)
+ group = yield self._txn.groupByUID(groupRecord.uid)
+ if group is not None:
+ members = yield self._txn.groupMembers(group.groupID)
groupCUAToAttendeeMemberPropMap[groupRecord.canonicalCalendarUserAddress()] = tuple(
[member.attendeeProperty(params={"MEMBER": groupCUA}) for member in sorted(members, key=lambda x: x.uid)]
)
@@ -2551,26 +2548,23 @@
groupUID = groupRecord.uid
else:
groupUID = uidFromCalendarUserAddress(groupCUA)
- (
- groupID, _ignore_name, membershipHash, _ignore_modDate,
- _ignore_extant
- ) = yield self._txn.groupByUID(groupUID)
+ group = yield self._txn.groupByUID(groupUID)
ga = schema.GROUP_ATTENDEE
- if groupID in groupIDToMembershipHashMap:
- if groupIDToMembershipHashMap[groupID] != membershipHash:
+ if group.groupID in groupIDToMembershipHashMap:
+ if groupIDToMembershipHashMap[group.groupID] != group.membershipHash:
yield Update(
- {ga.MEMBERSHIP_HASH: membershipHash, },
+ {ga.MEMBERSHIP_HASH: group.membershipHash, },
Where=(ga.RESOURCE_ID == self._resourceID).And(
- ga.GROUP_ID == groupID)
+ ga.GROUP_ID == group.groupID)
).on(self._txn)
changed = True
- del groupIDToMembershipHashMap[groupID]
+ del groupIDToMembershipHashMap[group.groupID]
else:
yield Insert({
ga.RESOURCE_ID: self._resourceID,
- ga.GROUP_ID: groupID,
- ga.MEMBERSHIP_HASH: membershipHash,
+ ga.GROUP_ID: group.groupID,
+ ga.MEMBERSHIP_HASH: group.membershipHash,
}).on(self._txn)
changed = True
@@ -5135,7 +5129,7 @@
setattr(self, attr, None)
- def externalize(self):
+ def serialize(self):
"""
Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
and reconstituted at the other end. Note that the other end may have a different schema so
@@ -5145,9 +5139,9 @@
@classmethod
- def internalize(cls, txn, mapping):
+ def deserialize(cls, txn, mapping):
"""
- Given a mapping generated by L{externalize}, convert the values into an array of database
+ Given a mapping generated by L{serialize}, convert the values into an array of database
like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
Note that there may be a schema mismatch with the external data, so treat missing items as
C{None} and ignore extra items.
@@ -5288,7 +5282,7 @@
returnValue(cls.makeClass(home._txn, rows[0]) if len(rows) == 1 else None)
- def externalize(self):
+ def serialize(self):
"""
Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
and reconstituted at the other end. Note that the other end may have a different schema so
@@ -5300,9 +5294,9 @@
@classmethod
- def internalize(cls, txn, mapping):
+ def deserialize(cls, txn, mapping):
"""
- Given a mapping generated by L{externalize}, convert the values into an array of database
+ Given a mapping generated by L{serialize}, convert the values into an array of database
like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
Note that there may be a schema mismatch with the external data, so treat missing items as
C{None} and ignore extra items.
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -69,7 +69,7 @@
Needed during migration.
"""
raw_results = yield self._txn.store().conduit.send_home_get_all_attachments(self)
- returnValue([Attachment.internalize(self._txn, attachment) for attachment in raw_results])
+ returnValue([Attachment.deserialize(self._txn, attachment) for attachment in raw_results])
@inlineCallbacks
@@ -89,7 +89,7 @@
Needed during migration only.
"""
raw_results = yield self._txn.store().conduit.send_home_get_attachment_links(self)
- returnValue([AttachmentLink.internalize(self._txn, attachment) for attachment in raw_results])
+ returnValue([AttachmentLink.deserialize(self._txn, attachment) for attachment in raw_results])
def getAllDropboxIDs(self):
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -196,5 +196,5 @@
# Calls on L{CommonHome} objects
-UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_all_attachments", "getAllAttachments", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
-UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_attachment_links", "getAttachmentLinks", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_all_attachments", "getAllAttachments", classMethod=False, transform_recv_result=UtilityConduitMixin._to_serialize_list)
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_attachment_links", "getAttachmentLinks", classMethod=False, transform_recv_result=UtilityConduitMixin._to_serialize_list)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -176,10 +176,10 @@
# TODO: Re-write attachment URIs - not sure if we need this as reverse proxy may take care of it
pass
- # TODO: group attendee reconcile
+ # TODO: shared collections reconcile
pass
- # TODO: shared collections reconcile
+ # TODO: group attendee reconcile
pass
# TODO: group sharee reconcile
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -133,8 +133,8 @@
# Calls on L{CommonHomeChild} objects
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_listobjects", "listObjects", classMethod=True)
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_movehere", "moveObjectResourceHere")
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_moveaway", "moveObjectResourceAway")
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_synctokenrevision", "syncTokenRevision")
@@ -142,14 +142,14 @@
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_search", "search")
# Calls on L{CommonObjectResource} objects
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize_list)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_listobjects", "listObjects", classMethod=True)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_countobjects", "countObjects", classMethod=True)
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_create", "create", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_create", "create", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_setcomponent", "setComponent")
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_component", "component", transform_recv_result=UtilityConduitMixin._to_string)
UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_remove", "remove")
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -104,7 +104,7 @@
from txdav.caldav.datastore.sql_external import CalendarHomeExternal
recipient = yield txn.store().directoryService().recordWithUID(uid)
- resourceID = yield txn.store().conduit.send_home_resource_id(self, recipient)
+ resourceID = yield txn.store().conduit.send_home_resource_id(txn, recipient)
home = CalendarHomeExternal(txn, recipient.uid, resourceID) if resourceID is not None else None
if home:
home._childClass = home._childClass._externalClass
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -212,19 +212,19 @@
# Transforms for returned data
#
@staticmethod
- def _to_externalize(value):
+ def _to_serialize(value):
"""
Convert the value to the external (JSON-based) representation.
"""
- return value.externalize() if value is not None else None
+ return value.serialize() if value is not None else None
@staticmethod
- def _to_externalize_list(value):
+ def _to_serialize_list(value):
"""
Convert the value to the external (JSON-based) representation.
"""
- return [v.externalize() for v in value]
+ return [v.serialize() for v in value]
@staticmethod
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -61,6 +61,8 @@
from txdav.carddav.iaddressbookstore import IAddressBookTransaction
from txdav.common.datastore.common import HomeChildBase
from txdav.common.datastore.podding.conduit import PoddingConduit
+from txdav.common.datastore.sql_directory import DelegatesAPIMixin, \
+ GroupsAPIMixin, GroupCacherAPIMixin
from txdav.common.datastore.sql_tables import _BIND_MODE_DIRECT, \
_BIND_MODE_INDIRECT, _BIND_MODE_OWN, _BIND_STATUS_ACCEPTED, \
_BIND_STATUS_DECLINED, _BIND_STATUS_DELETED, _BIND_STATUS_INVALID, \
@@ -68,7 +70,7 @@
_HOME_STATUS_PURGING, schema, splitSQLString, _HOME_STATUS_MIGRATING
from txdav.common.icommondatastore import ConcurrentModification, \
RecordNotAllowedError, ExternalShareFailed, ShareNotAllowed, \
- IndexedSearchException, NotFoundError
+ IndexedSearchException
from txdav.common.icommondatastore import HomeChildNameNotAllowedError, \
HomeChildNameAlreadyExistsError, NoSuchHomeChildError, \
ObjectResourceNameNotAllowedError, ObjectResourceNameAlreadyExistsError, \
@@ -80,7 +82,6 @@
from txdav.common.inotifications import INotificationCollection, \
INotificationObject
from txdav.idav import ChangeCategory
-from txdav.who.delegates import Delegates
from txdav.xml import element
from uuid import uuid4, UUID
@@ -88,7 +89,6 @@
from zope.interface import implements, directlyProvides
from collections import namedtuple
-import datetime
import inspect
import itertools
import json
@@ -565,7 +565,7 @@
-class CommonStoreTransaction(object):
+class CommonStoreTransaction(GroupsAPIMixin, GroupCacherAPIMixin, DelegatesAPIMixin):
"""
Transaction implementation for SQL database.
"""
@@ -1036,1033 +1036,6 @@
# End of IMIP
- # Groups
-
- @classproperty
- def _addGroupQuery(cls):
- gr = schema.GROUPS
- return Insert(
- {
- gr.NAME: Parameter("name"),
- gr.GROUP_UID: Parameter("groupUID"),
- gr.MEMBERSHIP_HASH: Parameter("membershipHash")
- },
- Return=gr.GROUP_ID
- )
-
-
- @classproperty
- def _updateGroupQuery(cls):
- gr = schema.GROUPS
- return Update(
- {
- gr.MEMBERSHIP_HASH: Parameter("membershipHash"),
- gr.NAME: Parameter("name"),
- gr.MODIFIED: Parameter("timestamp"),
- gr.EXTANT: Parameter("extant"),
- },
- Where=(gr.GROUP_UID == Parameter("groupUID"))
- )
-
-
- @classproperty
- def _groupByUID(cls):
- gr = schema.GROUPS
- return Select(
- [gr.GROUP_ID, gr.NAME, gr.MEMBERSHIP_HASH, gr.MODIFIED, gr.EXTANT],
- From=gr,
- Where=(gr.GROUP_UID == Parameter("groupUID"))
- )
-
-
- @classproperty
- def _groupByID(cls):
- gr = schema.GROUPS
- return Select(
- [gr.GROUP_UID, gr.NAME, gr.MEMBERSHIP_HASH, gr.EXTANT],
- From=gr,
- Where=(gr.GROUP_ID == Parameter("groupID"))
- )
-
-
- @classproperty
- def _deleteGroup(cls):
- gr = schema.GROUPS
- return Delete(
- From=gr,
- Where=(gr.GROUP_ID == Parameter("groupID"))
- )
-
-
- @inlineCallbacks
- def addGroup(self, groupUID, name, membershipHash):
- """
- @type groupUID: C{unicode}
- @type name: C{unicode}
- @type membershipHash: C{str}
- """
- record = yield self.directoryService().recordWithUID(groupUID)
- if record is None:
- returnValue(None)
-
- groupID = (yield self._addGroupQuery.on(
- self,
- name=name.encode("utf-8"),
- groupUID=groupUID.encode("utf-8"),
- membershipHash=membershipHash
- ))[0][0]
-
- yield self.refreshGroup(
- groupUID, record, groupID, name.encode("utf-8"), membershipHash, True
- )
- returnValue(groupID)
-
-
- def updateGroup(self, groupUID, name, membershipHash, extant=True):
- """
- @type groupUID: C{unicode}
- @type name: C{unicode}
- @type membershipHash: C{str}
- @type extant: C{boolean}
- """
- timestamp = datetime.datetime.utcnow()
- return self._updateGroupQuery.on(
- self,
- name=name.encode("utf-8"),
- groupUID=groupUID.encode("utf-8"),
- timestamp=timestamp,
- membershipHash=membershipHash,
- extant=(1 if extant else 0)
- )
-
-
- @inlineCallbacks
- def groupByUID(self, groupUID, create=True):
- """
- Return or create a record for the group UID.
-
- @type groupUID: C{unicode}
-
- @return: Deferred firing with tuple of group ID C{str}, group name
- C{unicode}, membership hash C{str}, modified timestamp, and
- extant C{boolean}
- """
- results = (
- yield self._groupByUID.on(
- self, groupUID=groupUID.encode("utf-8")
- )
- )
- if results:
- returnValue((
- results[0][0], # group id
- results[0][1].decode("utf-8"), # name
- results[0][2], # membership hash
- results[0][3], # modified timestamp
- bool(results[0][4]), # extant
- ))
- elif create:
- savepoint = SavepointAction("groupByUID")
- yield savepoint.acquire(self)
- try:
- groupID = yield self.addGroup(groupUID, u"", "")
- if groupID is None:
- # The record does not actually exist within the directory
- yield savepoint.release(self)
- returnValue((None, None, None, None, None))
-
- except Exception:
- yield savepoint.rollback(self)
- results = (
- yield self._groupByUID.on(
- self, groupUID=groupUID.encode("utf-8")
- )
- )
- if results:
- returnValue((
- results[0][0], # group id
- results[0][1].decode("utf-8"), # name
- results[0][2], # membership hash
- results[0][3], # modified timestamp
- bool(results[0][4]), # extant
- ))
- else:
- returnValue((None, None, None, None, None))
- else:
- yield savepoint.release(self)
- results = (
- yield self._groupByUID.on(
- self, groupUID=groupUID.encode("utf-8")
- )
- )
- if results:
- returnValue((
- results[0][0], # group id
- results[0][1].decode("utf-8"), # name
- results[0][2], # membership hash
- results[0][3], # modified timestamp
- bool(results[0][4]), # extant
- ))
- else:
- returnValue((None, None, None, None, None))
- else:
- returnValue((None, None, None, None, None))
-
-
- @inlineCallbacks
- def groupByID(self, groupID):
- """
- Given a group ID, return the group UID, or raise NotFoundError
-
- @type groupID: C{str}
- @return: Deferred firing with a tuple of group UID C{unicode},
- group name C{unicode}, membership hash C{str}, and extant C{boolean}
- """
- try:
- results = (yield self._groupByID.on(self, groupID=groupID))[0]
- if results:
- results = (
- results[0].decode("utf-8"),
- results[1].decode("utf-8"),
- results[2],
- bool(results[3])
- )
- returnValue(results)
- except IndexError:
- raise NotFoundError
-
-
- def deleteGroup(self, groupID):
- return self._deleteGroup.on(self, groupID=groupID)
-
- # End of Groups
-
-
- # Group Members
-
- @classproperty
- def _addMemberToGroupQuery(cls):
- gm = schema.GROUP_MEMBERSHIP
- return Insert(
- {
- gm.GROUP_ID: Parameter("groupID"),
- gm.MEMBER_UID: Parameter("memberUID")
- }
- )
-
-
- @classproperty
- def _removeMemberFromGroupQuery(cls):
- gm = schema.GROUP_MEMBERSHIP
- return Delete(
- From=gm,
- Where=(
- gm.GROUP_ID == Parameter("groupID")
- ).And(
- gm.MEMBER_UID == Parameter("memberUID")
- )
- )
-
-
- @classproperty
- def _selectGroupMembersQuery(cls):
- gm = schema.GROUP_MEMBERSHIP
- return Select(
- [gm.MEMBER_UID],
- From=gm,
- Where=(
- gm.GROUP_ID == Parameter("groupID")
- )
- )
-
-
- @classproperty
- def _selectGroupsForQuery(cls):
- gr = schema.GROUPS
- gm = schema.GROUP_MEMBERSHIP
-
- return Select(
- [gr.GROUP_UID],
- From=gr,
- Where=(
- gr.GROUP_ID.In(
- Select(
- [gm.GROUP_ID],
- From=gm,
- Where=(
- gm.MEMBER_UID == Parameter("uid")
- )
- )
- )
- )
- )
-
-
- def addMemberToGroup(self, memberUID, groupID):
- return self._addMemberToGroupQuery.on(
- self, groupID=groupID, memberUID=memberUID.encode("utf-8")
- )
-
-
- def removeMemberFromGroup(self, memberUID, groupID):
- return self._removeMemberFromGroupQuery.on(
- self, groupID=groupID, memberUID=memberUID.encode("utf-8")
- )
-
-
- @inlineCallbacks
- def groupMemberUIDs(self, groupID):
- """
- Returns the cached set of UIDs for members of the given groupID.
- Sub-groups are not returned in the results but their members are,
- because the group membership has already been expanded/flattened
- before storing in the db.
-
- @param groupID: the group ID
- @type groupID: C{int}
- @return: the set of member UIDs
- @rtype: a Deferred which fires with a set() of C{str} UIDs
- """
- members = set()
- results = (yield self._selectGroupMembersQuery.on(self, groupID=groupID))
- for row in results:
- members.add(row[0].decode("utf-8"))
- returnValue(members)
-
-
- @inlineCallbacks
- def refreshGroup(self, groupUID, record, groupID, cachedName, cachedMembershipHash, cachedExtant):
- """
- @param groupUID: the directory record
- @type groupUID: C{unicode}
- @param record: the directory record
- @type record: C{iDirectoryRecord}
- @param groupID: group resource id
- @type groupID: C{str}
- @param cachedName: group name in the database
- @type cachedName: C{unicode}
- @param cachedMembershipHash: membership hash in the database
- @type cachedMembershipHash: C{str}
- @param cachedExtant: extent field from in the database
- @type cachedExtant: C{bool}
-
- @return: Deferred firing with membershipChanged C{boolean}
-
- """
- if record is not None:
- memberUIDs = yield record.expandedMemberUIDs()
- name = record.displayName
- extant = True
- else:
- memberUIDs = frozenset()
- name = cachedName
- extant = False
-
- membershipHashContent = hashlib.md5()
- for memberUID in sorted(memberUIDs):
- membershipHashContent.update(str(memberUID))
- membershipHash = membershipHashContent.hexdigest()
-
- if cachedMembershipHash != membershipHash:
- membershipChanged = True
- log.debug(
- "Group '{group}' changed", group=name
- )
- else:
- membershipChanged = False
-
- if membershipChanged or extant != cachedExtant:
- # also updates group mod date
- yield self.updateGroup(
- groupUID, name, membershipHash, extant=extant
- )
-
- if membershipChanged:
- addedUIDs, removedUIDs = yield self.synchronizeMembers(groupID, set(memberUIDs))
- else:
- addedUIDs = removedUIDs = None
-
- returnValue((membershipChanged, addedUIDs, removedUIDs,))
-
-
- @inlineCallbacks
- def synchronizeMembers(self, groupID, newMemberUIDs):
- """
- Update the group membership table in the database to match the new membership list. This
- method will diff the existing set with the new set and apply the changes. It also calls out
- to a groupChanged() method with the set of added and removed members so that other modules
- that depend on groups can monitor the changes.
-
- @param groupID: group id of group to update
- @type groupID: L{str}
- @param newMemberUIDs: set of new member UIDs in the group
- @type newMemberUIDs: L{set} of L{str}
- """
- cachedMemberUIDs = (yield self.groupMemberUIDs(groupID))
-
- removed = cachedMemberUIDs - newMemberUIDs
- for memberUID in removed:
- yield self.removeMemberFromGroup(memberUID, groupID)
-
- added = newMemberUIDs - cachedMemberUIDs
- for memberUID in added:
- yield self.addMemberToGroup(memberUID, groupID)
-
- yield self.groupChanged(groupID, added, removed)
-
- returnValue((added, removed,))
-
-
- @inlineCallbacks
- def groupChanged(self, groupID, addedUIDs, removedUIDs):
- """
- Called when membership of a group changes.
-
- @param groupID: group id of group that changed
- @type groupID: L{str}
- @param addedUIDs: set of new member UIDs added to the group
- @type addedUIDs: L{set} of L{str}
- @param removedUIDs: set of old member UIDs removed from the group
- @type removedUIDs: L{set} of L{str}
- """
- yield Delegates.groupChanged(self, groupID, addedUIDs, removedUIDs)
-
-
- @inlineCallbacks
- def groupMembers(self, groupID):
- """
- The members of the given group as recorded in the db
- """
- members = set()
- memberUIDs = (yield self.groupMemberUIDs(groupID))
- for uid in memberUIDs:
- record = (yield self.directoryService().recordWithUID(uid))
- if record is not None:
- members.add(record)
- returnValue(members)
-
-
- @inlineCallbacks
- def groupUIDsFor(self, uid):
- """
- Returns the cached set of UIDs for the groups this given uid is
- a member of.
-
- @param uid: the uid
- @type uid: C{unicode}
- @return: the set of group IDs
- @rtype: a Deferred which fires with a set() of C{int} group IDs
- """
- groups = set()
- results = (
- yield self._selectGroupsForQuery.on(
- self, uid=uid.encode("utf-8")
- )
- )
- for row in results:
- groups.add(row[0].decode("utf-8"))
- returnValue(groups)
-
- # End of Group Members
-
- # Delegates
-
-
- @classproperty
- def _addDelegateQuery(cls):
- de = schema.DELEGATES
- return Insert({de.DELEGATOR: Parameter("delegator"),
- de.DELEGATE: Parameter("delegate"),
- de.READ_WRITE: Parameter("readWrite"),
- })
-
-
- @classproperty
- def _addDelegateGroupQuery(cls):
- ds = schema.DELEGATE_GROUPS
- return Insert({ds.DELEGATOR: Parameter("delegator"),
- ds.GROUP_ID: Parameter("groupID"),
- ds.READ_WRITE: Parameter("readWrite"),
- ds.IS_EXTERNAL: Parameter("isExternal"),
- })
-
-
- @classproperty
- def _removeDelegateQuery(cls):
- de = schema.DELEGATES
- return Delete(
- From=de,
- Where=(
- de.DELEGATOR == Parameter("delegator")
- ).And(
- de.DELEGATE == Parameter("delegate")
- ).And(
- de.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _removeDelegatesQuery(cls):
- de = schema.DELEGATES
- return Delete(
- From=de,
- Where=(
- de.DELEGATOR == Parameter("delegator")
- ).And(
- de.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _removeDelegateGroupQuery(cls):
- ds = schema.DELEGATE_GROUPS
- return Delete(
- From=ds,
- Where=(
- ds.DELEGATOR == Parameter("delegator")
- ).And(
- ds.GROUP_ID == Parameter("groupID")
- ).And(
- ds.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _removeDelegateGroupsQuery(cls):
- ds = schema.DELEGATE_GROUPS
- return Delete(
- From=ds,
- Where=(
- ds.DELEGATOR == Parameter("delegator")
- ).And(
- ds.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _selectDelegatesQuery(cls):
- de = schema.DELEGATES
- return Select(
- [de.DELEGATE],
- From=de,
- Where=(
- de.DELEGATOR == Parameter("delegator")
- ).And(
- de.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _selectDelegatorsToGroupQuery(cls):
- dg = schema.DELEGATE_GROUPS
- return Select(
- [dg.DELEGATOR],
- From=dg,
- Where=(
- dg.GROUP_ID == Parameter("delegateGroup")
- ).And(
- dg.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _selectDelegateGroupsQuery(cls):
- ds = schema.DELEGATE_GROUPS
- gr = schema.GROUPS
-
- return Select(
- [gr.GROUP_UID],
- From=gr,
- Where=(
- gr.GROUP_ID.In(
- Select(
- [ds.GROUP_ID],
- From=ds,
- Where=(
- ds.DELEGATOR == Parameter("delegator")
- ).And(
- ds.READ_WRITE == Parameter("readWrite")
- )
- )
- )
- )
- )
-
-
- @classproperty
- def _selectDirectDelegatorsQuery(cls):
- de = schema.DELEGATES
- return Select(
- [de.DELEGATOR],
- From=de,
- Where=(
- de.DELEGATE == Parameter("delegate")
- ).And(
- de.READ_WRITE == Parameter("readWrite")
- )
- )
-
-
- @classproperty
- def _selectIndirectDelegatorsQuery(cls):
- dg = schema.DELEGATE_GROUPS
- gm = schema.GROUP_MEMBERSHIP
-
- return Select(
- [dg.DELEGATOR],
- From=dg,
- Where=(
- dg.GROUP_ID.In(
- Select(
- [gm.GROUP_ID],
- From=gm,
- Where=(gm.MEMBER_UID == Parameter("delegate"))
- )
- ).And(
- dg.READ_WRITE == Parameter("readWrite")
- )
- )
- )
-
-
- @classproperty
- def _selectIndirectDelegatesQuery(cls):
- dg = schema.DELEGATE_GROUPS
- gm = schema.GROUP_MEMBERSHIP
-
- return Select(
- [gm.MEMBER_UID],
- From=gm,
- Where=(
- gm.GROUP_ID.In(
- Select(
- [dg.GROUP_ID],
- From=dg,
- Where=(dg.DELEGATOR == Parameter("delegator")).And(
- dg.READ_WRITE == Parameter("readWrite"))
- )
- )
- )
- )
-
-
- @classproperty
- def _selectExternalDelegateGroupsQuery(cls):
- edg = schema.EXTERNAL_DELEGATE_GROUPS
- return Select(
- [edg.DELEGATOR, edg.GROUP_UID_READ, edg.GROUP_UID_WRITE],
- From=edg
- )
-
-
- @classproperty
- def _removeExternalDelegateGroupsPairQuery(cls):
- edg = schema.EXTERNAL_DELEGATE_GROUPS
- return Delete(
- From=edg,
- Where=(
- edg.DELEGATOR == Parameter("delegator")
- )
- )
-
-
- @classproperty
- def _storeExternalDelegateGroupsPairQuery(cls):
- edg = schema.EXTERNAL_DELEGATE_GROUPS
- return Insert(
- {
- edg.DELEGATOR: Parameter("delegator"),
- edg.GROUP_UID_READ: Parameter("readDelegate"),
- edg.GROUP_UID_WRITE: Parameter("writeDelegate"),
- }
- )
-
-
- @classproperty
- def _removeExternalDelegateGroupsQuery(cls):
- ds = schema.DELEGATE_GROUPS
- return Delete(
- From=ds,
- Where=(
- ds.DELEGATOR == Parameter("delegator")
- ).And(
- ds.IS_EXTERNAL == 1
- )
- )
-
-
- @inlineCallbacks
- def addDelegate(self, delegator, delegate, readWrite):
- """
- Adds a row to the DELEGATES table. The delegate should not be a
- group. To delegate to a group, call addDelegateGroup() instead.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param delegate: the UID of the delegate
- @type delegate: C{unicode}
- @param readWrite: grant read and write access if True, otherwise
- read-only access
- @type readWrite: C{boolean}
- """
-
- def _addDelegate(subtxn):
- return self._addDelegateQuery.on(
- subtxn,
- delegator=delegator.encode("utf-8"),
- delegate=delegate.encode("utf-8"),
- readWrite=1 if readWrite else 0
- )
-
- try:
- yield self.subtransaction(_addDelegate, retries=0, failureOK=True)
- except AllRetriesFailed:
- pass
-
-
- @inlineCallbacks
- def addDelegateGroup(self, delegator, delegateGroupID, readWrite,
- isExternal=False):
- """
- Adds a row to the DELEGATE_GROUPS table. The delegate should be a
- group. To delegate to a person, call addDelegate() instead.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param delegateGroupID: the GROUP_ID of the delegate group
- @type delegateGroupID: C{int}
- @param readWrite: grant read and write access if True, otherwise
- read-only access
- @type readWrite: C{boolean}
- """
-
- def _addDelegateGroup(subtxn):
- return self._addDelegateGroupQuery.on(
- subtxn,
- delegator=delegator.encode("utf-8"),
- groupID=delegateGroupID,
- readWrite=1 if readWrite else 0,
- isExternal=1 if isExternal else 0
- )
-
- try:
- yield self.subtransaction(_addDelegateGroup, retries=0, failureOK=True)
- except AllRetriesFailed:
- pass
-
-
- def removeDelegate(self, delegator, delegate, readWrite):
- """
- Removes a row from the DELEGATES table. The delegate should not be a
- group. To remove a delegate group, call removeDelegateGroup() instead.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param delegate: the UID of the delegate
- @type delegate: C{unicode}
- @param readWrite: remove read and write access if True, otherwise
- read-only access
- @type readWrite: C{boolean}
- """
- return self._removeDelegateQuery.on(
- self,
- delegator=delegator.encode("utf-8"),
- delegate=delegate.encode("utf-8"),
- readWrite=1 if readWrite else 0
- )
-
-
- def removeDelegates(self, delegator, readWrite):
- """
- Removes all rows for this delegator/readWrite combination from the
- DELEGATES table.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param readWrite: remove read and write access if True, otherwise
- read-only access
- @type readWrite: C{boolean}
- """
- return self._removeDelegatesQuery.on(
- self,
- delegator=delegator.encode("utf-8"),
- readWrite=1 if readWrite else 0
- )
-
-
- def removeDelegateGroup(self, delegator, delegateGroupID, readWrite):
- """
- Removes a row from the DELEGATE_GROUPS table. The delegate should be a
- group. To remove a delegate person, call removeDelegate() instead.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param delegateGroupID: the GROUP_ID of the delegate group
- @type delegateGroupID: C{int}
- @param readWrite: remove read and write access if True, otherwise
- read-only access
- @type readWrite: C{boolean}
- """
- return self._removeDelegateGroupQuery.on(
- self,
- delegator=delegator.encode("utf-8"),
- groupID=delegateGroupID,
- readWrite=1 if readWrite else 0
- )
-
-
- def removeDelegateGroups(self, delegator, readWrite):
- """
- Removes all rows for this delegator/readWrite combination from the
- DELEGATE_GROUPS table.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param readWrite: remove read and write access if True, otherwise
- read-only access
- @type readWrite: C{boolean}
- """
- return self._removeDelegateGroupsQuery.on(
- self,
- delegator=delegator.encode("utf-8"),
- readWrite=1 if readWrite else 0
- )
-
-
- @inlineCallbacks
- def delegates(self, delegator, readWrite, expanded=False):
- """
- Returns the UIDs of all delegates for the given delegator. If
- expanded is False, only the direct delegates (users and groups)
- are returned. If expanded is True, the expanded membership is
- returned, not including the groups themselves.
-
- @param delegator: the UID of the delegator
- @type delegator: C{unicode}
- @param readWrite: the access-type to check for; read and write
- access if True, otherwise read-only access
- @type readWrite: C{boolean}
- @returns: the UIDs of the delegates (for the specified access
- type)
- @rtype: a Deferred resulting in a set
- """
- delegates = set()
- delegatorU = delegator.encode("utf-8")
-
- # First get the direct delegates
- results = (
- yield self._selectDelegatesQuery.on(
- self,
- delegator=delegatorU,
- readWrite=1 if readWrite else 0
- )
- )
- delegates.update([row[0].decode("utf-8") for row in results])
-
- if expanded:
- # Get those who are in groups which have been delegated to
- results = (
- yield self._selectIndirectDelegatesQuery.on(
- self,
- delegator=delegatorU,
- readWrite=1 if readWrite else 0
- )
- )
- # Skip the delegator if they are in one of the groups
- delegates.update([row[0].decode("utf-8") for row in results if row[0] != delegatorU])
-
- else:
- # Get the directly-delegated-to groups
- results = (
- yield self._selectDelegateGroupsQuery.on(
- self,
- delegator=delegatorU,
- readWrite=1 if readWrite else 0
- )
- )
- delegates.update([row[0].decode("utf-8") for row in results])
-
- returnValue(delegates)
-
-
- @inlineCallbacks
- def delegators(self, delegate, readWrite):
- """
- Returns the UIDs of all delegators which have granted access to
- the given delegate, either directly or indirectly via groups.
-
- @param delegate: the UID of the delegate
- @type delegate: C{unicode}
- @param readWrite: the access-type to check for; read and write
- access if True, otherwise read-only access
- @type readWrite: C{boolean}
- @returns: the UIDs of the delegators (for the specified access
- type)
- @rtype: a Deferred resulting in a set
- """
- delegators = set()
- delegateU = delegate.encode("utf-8")
-
- # First get the direct delegators
- results = (
- yield self._selectDirectDelegatorsQuery.on(
- self,
- delegate=delegateU,
- readWrite=1 if readWrite else 0
- )
- )
- delegators.update([row[0].decode("utf-8") for row in results])
-
- # Finally get those who have delegated to groups the delegate
- # is a member of
- results = (
- yield self._selectIndirectDelegatorsQuery.on(
- self,
- delegate=delegateU,
- readWrite=1 if readWrite else 0
- )
- )
- # Skip the delegator if they are in one of the groups
- delegators.update([row[0].decode("utf-8") for row in results if row[0] != delegateU])
-
- returnValue(delegators)
-
-
- @inlineCallbacks
- def delegatorsToGroup(self, delegateGroupID, readWrite):
- """
- Return the UIDs of those who have delegated to the given group with the
- given access level.
-
- @param delegateGroupID: the group ID of the delegate group
- @type delegateGroupID: C{int}
- @param readWrite: the access-type to check for; read and write
- access if True, otherwise read-only access
- @type readWrite: C{boolean}
- @returns: the UIDs of the delegators (for the specified access
- type)
- @rtype: a Deferred resulting in a set
-
- """
- results = (
- yield self._selectDelegatorsToGroupQuery.on(
- self,
- delegateGroup=delegateGroupID,
- readWrite=1 if readWrite else 0
- )
- )
- delegators = set([row[0].decode("utf-8") for row in results])
- returnValue(delegators)
-
-
- @inlineCallbacks
- def allGroupDelegates(self):
- """
- Return the UIDs of all groups which have been delegated to. Useful
- for obtaining the set of groups which need to be synchronized from
- the directory.
-
- @returns: the UIDs of all delegated-to groups
- @rtype: a Deferred resulting in a set
- """
- gr = schema.GROUPS
- dg = schema.DELEGATE_GROUPS
-
- results = (yield Select(
- [gr.GROUP_UID],
- From=gr,
- Where=(gr.GROUP_ID.In(Select([dg.GROUP_ID], From=dg, Where=None)))
- ).on(self))
- delegates = set()
- for row in results:
- delegates.add(row[0].decode("utf-8"))
-
- returnValue(delegates)
-
-
- @inlineCallbacks
- def externalDelegates(self):
- """
- Returns a dictionary mapping delegate UIDs to (read-group, write-group)
- tuples, including only those assignments that originated from the
- directory.
-
- @returns: dictionary mapping delegator uid to (readDelegateUID,
- writeDelegateUID) tuples
- @rtype: a Deferred resulting in a dictionary
- """
- delegates = {}
-
- # Get the externally managed delegates (which are all groups)
- results = (yield self._selectExternalDelegateGroupsQuery.on(self))
- for delegator, readDelegateUID, writeDelegateUID in results:
- delegates[delegator.encode("utf-8")] = (
- readDelegateUID.encode("utf-8") if readDelegateUID else None,
- writeDelegateUID.encode("utf-8") if writeDelegateUID else None
- )
-
- returnValue(delegates)
-
-
- @inlineCallbacks
- def assignExternalDelegates(
- self, delegator, readDelegateGroupID, writeDelegateGroupID,
- readDelegateUID, writeDelegateUID
- ):
- """
- Update the external delegate group table so we can quickly identify
- diffs next time, and update the delegate group table itself
-
- @param delegator
- @type delegator: C{UUID}
- """
-
- # Delete existing external assignments for the delegator
- yield self._removeExternalDelegateGroupsQuery.on(
- self,
- delegator=str(delegator)
- )
-
- # Remove from the external comparison table
- yield self._removeExternalDelegateGroupsPairQuery.on(
- self,
- delegator=str(delegator)
- )
-
- # Store new assignments in the external comparison table
- if readDelegateUID or writeDelegateUID:
- readDelegateForDB = (
- readDelegateUID.encode("utf-8") if readDelegateUID else ""
- )
- writeDelegateForDB = (
- writeDelegateUID.encode("utf-8") if writeDelegateUID else ""
- )
- yield self._storeExternalDelegateGroupsPairQuery.on(
- self,
- delegator=str(delegator),
- readDelegate=readDelegateForDB,
- writeDelegate=writeDelegateForDB
- )
-
- # Apply new assignments
- if readDelegateGroupID is not None:
- yield self.addDelegateGroup(
- delegator, readDelegateGroupID, False, isExternal=True
- )
- if writeDelegateGroupID is not None:
- yield self.addDelegateGroup(
- delegator, writeDelegateGroupID, True, isExternal=True
- )
-
-
- # End of Delegates
-
-
def preCommit(self, operation):
"""
Run things before C{commit}. (Note: only provided by SQL
@@ -6033,7 +5006,7 @@
return self.ownerHome().externalClass()
- def externalize(self):
+ def serialize(self):
"""
Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
and reconstituted at the other end. Note that the other end may have a different schema so
@@ -6048,9 +5021,9 @@
@classmethod
@inlineCallbacks
- def internalize(cls, parent, mapping):
+ def deserialize(cls, parent, mapping):
"""
- Given a mapping generated by L{externalize}, convert the values into an array of database
+ Given a mapping generated by L{serialize}, convert the values into an array of database
like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
Note that there may be a schema mismatch with the external data, so treat missing items as
C{None} and ignore extra items.
@@ -7290,7 +6263,7 @@
)
- def externalize(self):
+ def serialize(self):
"""
Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
and reconstituted at the other end. Note that the other end may have a different schema so
@@ -7301,9 +6274,9 @@
@classmethod
@inlineCallbacks
- def internalize(cls, parent, mapping):
+ def deserialize(cls, parent, mapping):
"""
- Given a mapping generated by L{externalize}, convert the values into an array of database
+ Given a mapping generated by L{serialize}, convert the values into an array of database
like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
Note that there may be a schema mismatch with the external data, so treat missing items as
C{None} and ignore extra items.
Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_directory.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_directory.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -0,0 +1,773 @@
+# -*- test-case-name: twext.enterprise.dal.test.test_record -*-
+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.record import SerializableRecord, fromTable
+from twext.enterprise.dal.syntax import SavepointAction
+from twext.python.log import Logger
+from twisted.internet.defer import inlineCallbacks, returnValue
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.icommondatastore import AllRetriesFailed, NotFoundError
+import datetime
+import hashlib
+from txdav.who.delegates import Delegates
+
+log = Logger()
+
+"""
+Classes and methods that relate to directory objects in the SQL store. e.g.,
+delegates, groups etc
+"""
+
+class GroupsRecord(SerializableRecord, fromTable(schema.GROUPS)):
+ """
+ @DynamicAttrs
+ L{Record} for L{schema.GROUPS}.
+ """
+
+ @classmethod
+ def groupsForMember(cls, txn, memberUID):
+
+ return GroupsRecord.query(
+ txn,
+ GroupsRecord.groupID.In(
+ GroupMembershipRecord.queryExpr(
+ GroupMembershipRecord.memberUID == memberUID.encode("utf-8"),
+ attributes=(GroupMembershipRecord.groupID,),
+ )
+ ),
+ )
+
+
+
+class GroupMembershipRecord(SerializableRecord, fromTable(schema.GROUP_MEMBERSHIP)):
+ """
+ @DynamicAttrs
+ L{Record} for L{schema.GROUP_MEMBERSHIP}.
+ """
+ pass
+
+
+
+class DelegateRecord(SerializableRecord, fromTable(schema.DELEGATES)):
+ """
+ @DynamicAttrs
+ L{Record} for L{schema.DELEGATES}.
+ """
+ pass
+
+
+
+class DelegateGroupsRecord(SerializableRecord, fromTable(schema.DELEGATE_GROUPS)):
+ """
+ @DynamicAttrs
+ L{Record} for L{schema.DELEGATE_GROUPS}.
+ """
+
+ @classmethod
+ def allGroupDelegates(cls, txn):
+ """
+ Get the directly-delegated-to groups.
+ """
+
+ return GroupsRecord.query(
+ txn,
+ GroupsRecord.groupID.In(
+ DelegateGroupsRecord.queryExpr(
+ None,
+ attributes=(DelegateGroupsRecord.groupID,),
+ )
+ ),
+ )
+
+
+ @classmethod
+ def delegateGroups(cls, txn, delegator, readWrite):
+ """
+ Get the directly-delegated-to groups.
+ """
+
+ return GroupsRecord.query(
+ txn,
+ GroupsRecord.groupID.In(
+ DelegateGroupsRecord.queryExpr(
+ (DelegateGroupsRecord.delegator == delegator.encode("utf-8")).And(
+ DelegateGroupsRecord.readWrite == (1 if readWrite else 0)
+ ),
+ attributes=(DelegateGroupsRecord.groupID,),
+ )
+ ),
+ )
+
+
+ @classmethod
+ def indirectDelegators(cls, txn, delegate, readWrite):
+ """
+ Get delegators who have delegated to groups the delegate is a member of.
+ """
+
+ return cls.query(
+ txn,
+ cls.groupID.In(
+ GroupMembershipRecord.queryExpr(
+ GroupMembershipRecord.memberUID == delegate.encode("utf-8"),
+ attributes=(GroupMembershipRecord.groupID,),
+ )
+ ).And(cls.readWrite == (1 if readWrite else 0)),
+ )
+
+
+ @classmethod
+ def indirectDelegates(cls, txn, delegator, readWrite):
+ """
+ Get delegates who are in groups which have been delegated to.
+ """
+
+ return GroupMembershipRecord.query(
+ txn,
+ GroupMembershipRecord.groupID.In(
+ DelegateGroupsRecord.queryExpr(
+ (DelegateGroupsRecord.delegator == delegator.encode("utf-8")).And(
+ DelegateGroupsRecord.readWrite == (1 if readWrite else 0)
+ ),
+ attributes=(DelegateGroupsRecord.groupID,),
+ )
+ ),
+ )
+
+
+
+class ExternalDelegateGroupsRecord(SerializableRecord, fromTable(schema.EXTERNAL_DELEGATE_GROUPS)):
+ """
+ @DynamicAttrs
+ L{Record} for L{schema.EXTERNAL_DELEGATE_GROUPS}.
+ """
+ pass
+
+
+
+class GroupsAPIMixin(object):
+ """
+ A mixin for L{CommonStoreTransaction} that covers the groups API.
+ """
+
+ @inlineCallbacks
+ def addGroup(self, groupUID, name, membershipHash):
+ """
+ @type groupUID: C{unicode}
+ @type name: C{unicode}
+ @type membershipHash: C{str}
+ """
+ record = yield self.directoryService().recordWithUID(groupUID)
+ if record is None:
+ returnValue(None)
+
+ group = yield GroupsRecord.create(
+ self,
+ name=name.encode("utf-8"),
+ groupUID=groupUID.encode("utf-8"),
+ membershipHash=membershipHash,
+ )
+
+ yield self.refreshGroup(group, record)
+ returnValue(group)
+
+
+ def updateGroup(self, groupUID, name, membershipHash, extant=True):
+ """
+ @type groupUID: C{unicode}
+ @type name: C{unicode}
+ @type membershipHash: C{str}
+ @type extant: C{boolean}
+ """
+ timestamp = datetime.datetime.utcnow()
+ group = yield self.groupByUID(groupUID, create=False)
+ if group is not None:
+ yield group.update(
+ name=name.encode("utf-8"),
+ membershipHash=membershipHash,
+ extant=(1 if extant else 0),
+ modified=timestamp,
+ )
+
+
+ @inlineCallbacks
+ def groupByUID(self, groupUID, create=True):
+ """
+ Return or create a record for the group UID.
+
+ @type groupUID: C{unicode}
+
+ @return: Deferred firing with tuple of group ID C{str}, group name
+ C{unicode}, membership hash C{str}, modified timestamp, and
+ extant C{boolean}
+ """
+ results = yield GroupsRecord.query(
+ self,
+ GroupsRecord.groupUID == groupUID.encode("utf-8")
+ )
+ if results:
+ returnValue(results[0])
+ elif create:
+ savepoint = SavepointAction("groupByUID")
+ yield savepoint.acquire(self)
+ try:
+ group = yield self.addGroup(groupUID, u"", "")
+ if group is None:
+ # The record does not actually exist within the directory
+ yield savepoint.release(self)
+ returnValue(None)
+
+ except Exception:
+ yield savepoint.rollback(self)
+ results = yield GroupsRecord.query(
+ self,
+ GroupsRecord.groupUID == groupUID.encode("utf-8")
+ )
+ returnValue(results[0] if results else None)
+ else:
+ yield savepoint.release(self)
+ returnValue(group)
+ else:
+ returnValue(None)
+
+
+ @inlineCallbacks
+ def groupByID(self, groupID):
+ """
+ Given a group ID, return the group UID, or raise NotFoundError
+
+ @type groupID: C{str}
+ @return: Deferred firing with a tuple of group UID C{unicode},
+ group name C{unicode}, membership hash C{str}, and extant C{boolean}
+ """
+ results = yield GroupsRecord.query(
+ self,
+ GroupsRecord.groupID == groupID,
+ )
+ if results:
+ returnValue(results[0])
+ else:
+ raise NotFoundError
+
+
+
+class GroupCacherAPIMixin(object):
+ """
+ A mixin for L{CommonStoreTransaction} that covers the group cacher API.
+ """
+
+ def addMemberToGroup(self, memberUID, groupID):
+ return GroupMembershipRecord.create(self, groupID=groupID, memberUID=memberUID.encode("utf-8"))
+
+
+ def removeMemberFromGroup(self, memberUID, groupID):
+ return GroupMembershipRecord.deletematch(
+ self, groupID=groupID, memberUID=memberUID.encode("utf-8")
+ )
+
+
+ @inlineCallbacks
+ def groupMemberUIDs(self, groupID):
+ """
+ Returns the cached set of UIDs for members of the given groupID.
+ Sub-groups are not returned in the results but their members are,
+ because the group membership has already been expanded/flattened
+ before storing in the db.
+
+ @param groupID: the group ID
+ @type groupID: C{int}
+ @return: the set of member UIDs
+ @rtype: a Deferred which fires with a set() of C{str} UIDs
+ """
+
+ members = yield GroupMembershipRecord.query(self, GroupMembershipRecord.groupID == groupID)
+ returnValue(set([record.memberUID.decode("utf-8") for record in members]))
+
+
+ @inlineCallbacks
+ def refreshGroup(self, group, record):
+ """
+ @param group: the group record
+ @type group: L{GroupsRecord}
+ @param record: the directory record
+ @type record: C{iDirectoryRecord}
+
+ @return: Deferred firing with membershipChanged C{boolean}
+
+ """
+
+ if record is not None:
+ memberUIDs = yield record.expandedMemberUIDs()
+ name = record.displayName
+ extant = True
+ else:
+ memberUIDs = frozenset()
+ name = group.name
+ extant = False
+
+ membershipHashContent = hashlib.md5()
+ for memberUID in sorted(memberUIDs):
+ membershipHashContent.update(str(memberUID))
+ membershipHash = membershipHashContent.hexdigest()
+
+ if group.membershipHash != membershipHash:
+ membershipChanged = True
+ log.debug(
+ "Group '{group}' changed", group=name
+ )
+ else:
+ membershipChanged = False
+
+ if membershipChanged or extant != group.extant:
+ # also updates group mod date
+ yield group.update(
+ name=name,
+ membershipHash=membershipHash,
+ extant=(1 if extant else 0),
+ )
+
+ if membershipChanged:
+ addedUIDs, removedUIDs = yield self.synchronizeMembers(group.groupID, set(memberUIDs))
+ else:
+ addedUIDs = removedUIDs = None
+
+ returnValue((membershipChanged, addedUIDs, removedUIDs,))
+
+
+ @inlineCallbacks
+ def synchronizeMembers(self, groupID, newMemberUIDs):
+ """
+ Update the group membership table in the database to match the new membership list. This
+ method will diff the existing set with the new set and apply the changes. It also calls out
+ to a groupChanged() method with the set of added and removed members so that other modules
+ that depend on groups can monitor the changes.
+
+ @param groupID: group id of group to update
+ @type groupID: L{str}
+ @param newMemberUIDs: set of new member UIDs in the group
+ @type newMemberUIDs: L{set} of L{str}
+ """
+ cachedMemberUIDs = yield self.groupMemberUIDs(groupID)
+
+ removed = cachedMemberUIDs - newMemberUIDs
+ for memberUID in removed:
+ yield self.removeMemberFromGroup(memberUID, groupID)
+
+ added = newMemberUIDs - cachedMemberUIDs
+ for memberUID in added:
+ yield self.addMemberToGroup(memberUID, groupID)
+
+ yield self.groupChanged(groupID, added, removed)
+
+ returnValue((added, removed,))
+
+
+ @inlineCallbacks
+ def groupChanged(self, groupID, addedUIDs, removedUIDs):
+ """
+ Called when membership of a group changes.
+
+ @param groupID: group id of group that changed
+ @type groupID: L{str}
+ @param addedUIDs: set of new member UIDs added to the group
+ @type addedUIDs: L{set} of L{str}
+ @param removedUIDs: set of old member UIDs removed from the group
+ @type removedUIDs: L{set} of L{str}
+ """
+ yield Delegates.groupChanged(self, groupID, addedUIDs, removedUIDs)
+
+
+ @inlineCallbacks
+ def groupMembers(self, groupID):
+ """
+ The members of the given group as recorded in the db
+ """
+ members = set()
+ memberUIDs = (yield self.groupMemberUIDs(groupID))
+ for uid in memberUIDs:
+ record = (yield self.directoryService().recordWithUID(uid))
+ if record is not None:
+ members.add(record)
+ returnValue(members)
+
+
+ @inlineCallbacks
+ def groupUIDsFor(self, uid):
+ """
+ Returns the cached set of UIDs for the groups this given uid is
+ a member of.
+
+ @param uid: the uid
+ @type uid: C{unicode}
+ @return: the set of group IDs
+ @rtype: a Deferred which fires with a set() of C{int} group IDs
+ """
+ groups = yield GroupsRecord.groupsForMember(self, uid)
+ returnValue(set([group.groupUID.decode("utf-8") for group in groups]))
+
+
+
+class DelegatesAPIMixin(object):
+ """
+ A mixin for L{CommonStoreTransaction} that covers the delegates API.
+ """
+
+ @inlineCallbacks
+ def addDelegate(self, delegator, delegate, readWrite):
+ """
+ Adds a row to the DELEGATES table. The delegate should not be a
+ group. To delegate to a group, call addDelegateGroup() instead.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param delegate: the UID of the delegate
+ @type delegate: C{unicode}
+ @param readWrite: grant read and write access if True, otherwise
+ read-only access
+ @type readWrite: C{boolean}
+ """
+
+ def _addDelegate(subtxn):
+ return DelegateRecord.create(
+ subtxn,
+ delegator=delegator.encode("utf-8"),
+ delegate=delegate.encode("utf-8"),
+ readWrite=1 if readWrite else 0
+ )
+
+ try:
+ yield self.subtransaction(_addDelegate, retries=0, failureOK=True)
+ except AllRetriesFailed:
+ pass
+
+
+ @inlineCallbacks
+ def addDelegateGroup(self, delegator, delegateGroupID, readWrite,
+ isExternal=False):
+ """
+ Adds a row to the DELEGATE_GROUPS table. The delegate should be a
+ group. To delegate to a person, call addDelegate() instead.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param delegateGroupID: the GROUP_ID of the delegate group
+ @type delegateGroupID: C{int}
+ @param readWrite: grant read and write access if True, otherwise
+ read-only access
+ @type readWrite: C{boolean}
+ """
+
+ def _addDelegateGroup(subtxn):
+ return DelegateGroupsRecord.create(
+ subtxn,
+ delegator=delegator.encode("utf-8"),
+ groupID=delegateGroupID,
+ readWrite=1 if readWrite else 0,
+ isExternal=1 if isExternal else 0
+ )
+
+ try:
+ yield self.subtransaction(_addDelegateGroup, retries=0, failureOK=True)
+ except AllRetriesFailed:
+ pass
+
+
+ def removeDelegate(self, delegator, delegate, readWrite):
+ """
+ Removes a row from the DELEGATES table. The delegate should not be a
+ group. To remove a delegate group, call removeDelegateGroup() instead.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param delegate: the UID of the delegate
+ @type delegate: C{unicode}
+ @param readWrite: remove read and write access if True, otherwise
+ read-only access
+ @type readWrite: C{boolean}
+ """
+ return DelegateRecord.deletesome(
+ self,
+ (DelegateRecord.delegator == delegator.encode("utf-8")).And(
+ DelegateRecord.delegate == delegate.encode("utf-8")).And(
+ DelegateRecord.readWrite == (1 if readWrite else 0)
+ ),
+ )
+
+
+ def removeDelegates(self, delegator, readWrite):
+ """
+ Removes all rows for this delegator/readWrite combination from the
+ DELEGATES table.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param readWrite: remove read and write access if True, otherwise
+ read-only access
+ @type readWrite: C{boolean}
+ """
+ return DelegateRecord.deletesome(
+ self,
+ (DelegateRecord.delegator == delegator.encode("utf-8")).And(
+ DelegateRecord.readWrite == (1 if readWrite else 0)
+ ),
+ )
+
+
+ def removeDelegateGroup(self, delegator, delegateGroupID, readWrite):
+ """
+ Removes a row from the DELEGATE_GROUPS table. The delegate should be a
+ group. To remove a delegate person, call removeDelegate() instead.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param delegateGroupID: the GROUP_ID of the delegate group
+ @type delegateGroupID: C{int}
+ @param readWrite: remove read and write access if True, otherwise
+ read-only access
+ @type readWrite: C{boolean}
+ """
+ return DelegateGroupsRecord.deletesome(
+ self,
+ (DelegateGroupsRecord.delegator == delegator.encode("utf-8")).And(
+ DelegateGroupsRecord.groupID == delegateGroupID).And(
+ DelegateGroupsRecord.readWrite == (1 if readWrite else 0)
+ ),
+ )
+
+
+ def removeDelegateGroups(self, delegator, readWrite):
+ """
+ Removes all rows for this delegator/readWrite combination from the
+ DELEGATE_GROUPS table.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param readWrite: remove read and write access if True, otherwise
+ read-only access
+ @type readWrite: C{boolean}
+ """
+ return DelegateGroupsRecord.deletesome(
+ self,
+ (DelegateGroupsRecord.delegator == delegator.encode("utf-8")).And(
+ DelegateGroupsRecord.readWrite == (1 if readWrite else 0)
+ ),
+ )
+
+
+ @inlineCallbacks
+ def delegates(self, delegator, readWrite, expanded=False):
+ """
+ Returns the UIDs of all delegates for the given delegator. If
+ expanded is False, only the direct delegates (users and groups)
+ are returned. If expanded is True, the expanded membership is
+ returned, not including the groups themselves.
+
+ @param delegator: the UID of the delegator
+ @type delegator: C{unicode}
+ @param readWrite: the access-type to check for; read and write
+ access if True, otherwise read-only access
+ @type readWrite: C{boolean}
+ @returns: the UIDs of the delegates (for the specified access
+ type)
+ @rtype: a Deferred resulting in a set
+ """
+ delegates = set()
+ delegatorU = delegator.encode("utf-8")
+
+ # First get the direct delegates
+ results = yield DelegateRecord.query(
+ self,
+ (DelegateRecord.delegator == delegatorU).And(
+ DelegateRecord.readWrite == (1 if readWrite else 0)
+ )
+ )
+ delegates.update([record.delegate.decode("utf-8") for record in results])
+
+ if expanded:
+ # Get those who are in groups which have been delegated to
+ results = yield DelegateGroupsRecord.indirectDelegates(
+ self, delegator, readWrite
+ )
+ # Skip the delegator if they are in one of the groups
+ delegates.update([record.memberUID.decode("utf-8") for record in results if record.memberUID != delegatorU])
+
+ else:
+ # Get the directly-delegated-to groups
+ results = yield DelegateGroupsRecord.delegateGroups(
+ self, delegator, readWrite,
+ )
+ delegates.update([record.groupUID.decode("utf-8") for record in results])
+
+ returnValue(delegates)
+
+
+ @inlineCallbacks
+ def delegators(self, delegate, readWrite):
+ """
+ Returns the UIDs of all delegators which have granted access to
+ the given delegate, either directly or indirectly via groups.
+
+ @param delegate: the UID of the delegate
+ @type delegate: C{unicode}
+ @param readWrite: the access-type to check for; read and write
+ access if True, otherwise read-only access
+ @type readWrite: C{boolean}
+ @returns: the UIDs of the delegators (for the specified access
+ type)
+ @rtype: a Deferred resulting in a set
+ """
+ delegators = set()
+ delegateU = delegate.encode("utf-8")
+
+ # First get the direct delegators
+ results = yield DelegateRecord.query(
+ self,
+ (DelegateRecord.delegate == delegateU).And(
+ DelegateRecord.readWrite == (1 if readWrite else 0)
+ )
+ )
+ delegators.update([record.delegator.decode("utf-8") for record in results])
+
+ # Finally get those who have delegated to groups the delegate
+ # is a member of
+ results = yield DelegateGroupsRecord.indirectDelegators(
+ self, delegate, readWrite
+ )
+ # Skip the delegator if they are in one of the groups
+ delegators.update([record.delegator.decode("utf-8") for record in results if record.delegator != delegateU])
+
+ returnValue(delegators)
+
+
+ @inlineCallbacks
+ def delegatorsToGroup(self, delegateGroupID, readWrite):
+ """
+ Return the UIDs of those who have delegated to the given group with the
+ given access level.
+
+ @param delegateGroupID: the group ID of the delegate group
+ @type delegateGroupID: C{int}
+ @param readWrite: the access-type to check for; read and write
+ access if True, otherwise read-only access
+ @type readWrite: C{boolean}
+ @returns: the UIDs of the delegators (for the specified access
+ type)
+ @rtype: a Deferred resulting in a set
+
+ """
+ results = yield DelegateGroupsRecord.query(
+ self,
+ (DelegateGroupsRecord.groupID == delegateGroupID).And(
+ DelegateGroupsRecord.readWrite == (1 if readWrite else 0)
+ )
+ )
+ delegators = set([record.delegator.decode("utf-8") for record in results])
+ returnValue(delegators)
+
+
+ @inlineCallbacks
+ def allGroupDelegates(self):
+ """
+ Return the UIDs of all groups which have been delegated to. Useful
+ for obtaining the set of groups which need to be synchronized from
+ the directory.
+
+ @returns: the UIDs of all delegated-to groups
+ @rtype: a Deferred resulting in a set
+ """
+
+ results = yield DelegateGroupsRecord.allGroupDelegates(self)
+ delegates = set([record.groupUID.decode("utf-8") for record in results])
+
+ returnValue(delegates)
+
+
+ @inlineCallbacks
+ def externalDelegates(self):
+ """
+ Returns a dictionary mapping delegate UIDs to (read-group, write-group)
+ tuples, including only those assignments that originated from the
+ directory.
+
+ @returns: dictionary mapping delegator uid to (readDelegateUID,
+ writeDelegateUID) tuples
+ @rtype: a Deferred resulting in a dictionary
+ """
+ delegates = {}
+
+ # Get the externally managed delegates (which are all groups)
+ results = yield ExternalDelegateGroupsRecord.all(self)
+ for record in results:
+ delegates[record.delegator.encode("utf-8")] = (
+ record.groupUIDRead.encode("utf-8") if record.groupUIDRead else None,
+ record.groupUIDWrite.encode("utf-8") if record.groupUIDWrite else None
+ )
+
+ returnValue(delegates)
+
+
+ @inlineCallbacks
+ def assignExternalDelegates(
+ self, delegator, readDelegateGroupID, writeDelegateGroupID,
+ readDelegateUID, writeDelegateUID
+ ):
+ """
+ Update the external delegate group table so we can quickly identify
+ diffs next time, and update the delegate group table itself
+
+ @param delegator
+ @type delegator: C{UUID}
+ """
+
+ # Delete existing external assignments for the delegator
+ yield DelegateGroupsRecord.deletesome(
+ self,
+ (DelegateGroupsRecord.delegator == str(delegator)).And(
+ DelegateGroupsRecord.isExternal == 1
+ )
+ )
+
+ # Remove from the external comparison table
+ yield ExternalDelegateGroupsRecord.deletesome(
+ self,
+ ExternalDelegateGroupsRecord.delegator == str(delegator)
+ )
+
+ # Store new assignments in the external comparison table
+ if readDelegateUID or writeDelegateUID:
+ readDelegateForDB = (
+ readDelegateUID.encode("utf-8") if readDelegateUID else ""
+ )
+ writeDelegateForDB = (
+ writeDelegateUID.encode("utf-8") if writeDelegateUID else ""
+ )
+ yield ExternalDelegateGroupsRecord.create(
+ self,
+ delegator=str(delegator),
+ groupUIDRead=readDelegateForDB,
+ groupUIDWrite=writeDelegateForDB,
+ )
+
+ # Apply new assignments
+ if readDelegateGroupID is not None:
+ yield self.addDelegateGroup(
+ delegator, readDelegateGroupID, False, isExternal=True
+ )
+ if writeDelegateGroupID is not None:
+ yield self.addDelegateGroup(
+ delegator, writeDelegateGroupID, True, isExternal=True
+ )
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -209,7 +209,7 @@
results = []
for mapping in raw_results:
- child = yield cls.internalize(home, mapping)
+ child = yield cls.deserialize(home, mapping)
results.append(child)
returnValue(results)
@@ -220,7 +220,7 @@
mapping = yield home._txn.store().conduit.send_homechild_objectwith(home, name, resourceID, externalID, accepted)
if mapping:
- child = yield cls.internalize(home, mapping)
+ child = yield cls.deserialize(home, mapping)
returnValue(child)
else:
returnValue(None)
@@ -357,7 +357,7 @@
results = []
if mapping_list:
for mapping in mapping_list:
- child = yield cls.internalize(parent, mapping)
+ child = yield cls.deserialize(parent, mapping)
results.append(child)
returnValue(results)
@@ -370,7 +370,7 @@
results = []
if mapping_list:
for mapping in mapping_list:
- child = yield cls.internalize(parent, mapping)
+ child = yield cls.deserialize(parent, mapping)
results.append(child)
returnValue(results)
@@ -395,7 +395,7 @@
mapping = yield parent._txn.store().conduit.send_objectresource_objectwith(parent, name, uid, resourceID)
if mapping:
- child = yield cls.internalize(parent, mapping)
+ child = yield cls.deserialize(parent, mapping)
returnValue(child)
else:
returnValue(None)
@@ -421,7 +421,7 @@
mapping = yield parent._txn.store().conduit.send_objectresource_create(parent, name, str(component), options=options)
if mapping:
- child = yield cls.internalize(parent, mapping)
+ child = yield cls.deserialize(parent, mapping)
returnValue(child)
else:
returnValue(None)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/delegates.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/delegates.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/delegates.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -353,13 +353,8 @@
if delegate.recordType == BaseRecordType.group:
# find the groupID
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- _ignore_extant
- ) = yield txn.groupByUID(
- delegate.uid
- )
- yield txn.addDelegateGroup(delegator.uid, groupID, readWrite)
+ group = yield txn.groupByUID(delegate.uid)
+ yield txn.addDelegateGroup(delegator.uid, group.groupID, readWrite)
else:
yield txn.addDelegate(delegator.uid, delegate.uid, readWrite)
@@ -393,13 +388,8 @@
if delegate.recordType == BaseRecordType.group:
# find the groupID
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- _ignore_extant
- ) = yield txn.groupByUID(
- delegate.uid
- )
- yield txn.removeDelegateGroup(delegator.uid, groupID, readWrite)
+ group = yield txn.groupByUID(delegate.uid)
+ yield txn.removeDelegateGroup(delegator.uid, group.groupID, readWrite)
else:
yield txn.removeDelegate(delegator.uid, delegate.uid, readWrite)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -85,7 +85,7 @@
class GroupRefreshWork(AggregatedWorkItem, fromTable(schema.GROUP_REFRESH_WORK)):
- group = property(lambda self: (self.table.GROUP_UID == self.groupUid))
+ group = property(lambda self: (self.table.GROUP_UID == self.groupUID))
@inlineCallbacks
def doWork(self):
@@ -94,27 +94,27 @@
try:
yield groupCacher.refreshGroup(
- self.transaction, self.groupUid.decode("utf-8")
+ self.transaction, self.groupUID.decode("utf-8")
)
except Exception, e:
log.error(
"Failed to refresh group {group} {err}",
- group=self.groupUid, err=e
+ group=self.groupUID, err=e
)
else:
log.debug(
"Rescheduling group refresh for {group}: {when}",
- group=self.groupUid,
+ group=self.groupUID,
when=datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
)
- yield self.reschedule(self.transaction, 10, groupUID=self.groupUid)
+ yield self.reschedule(self.transaction, 10, groupUID=self.groupUID)
class GroupDelegateChangesWork(AggregatedWorkItem, fromTable(schema.GROUP_DELEGATE_CHANGES_WORK)):
- delegator = property(lambda self: (self.table.DELEGATOR_UID == self.delegatorUid))
+ delegator = property(lambda self: (self.table.DELEGATOR_UID == self.delegatorUID))
@inlineCallbacks
def doWork(self):
@@ -124,14 +124,14 @@
try:
yield groupCacher.applyExternalAssignments(
self.transaction,
- self.delegatorUid.decode("utf-8"),
- self.readDelegateUid.decode("utf-8"),
- self.writeDelegateUid.decode("utf-8")
+ self.delegatorUID.decode("utf-8"),
+ self.readDelegateUID.decode("utf-8"),
+ self.writeDelegateUID.decode("utf-8")
)
except Exception, e:
log.error(
"Failed to apply external delegates for {uid} {err}",
- uid=self.delegatorUid, err=e
+ uid=self.delegatorUID, err=e
)
@@ -182,8 +182,8 @@
homeID = rows[0][0]
home = yield self.transaction.calendarHomeWithResourceID(homeID)
calendar = yield home.childWithID(self.calendarID)
- groupUID = ((yield self.transaction.groupByID(self.groupID)))[0]
- yield calendar.reconcileGroupSharee(groupUID)
+ group = (yield self.transaction.groupByID(self.groupID))
+ yield calendar.reconcileGroupSharee(group.groupUID)
@@ -302,7 +302,7 @@
# For each of those groups, create a per-group refresh work item
for groupUID in set(groupUIDs) - set(deletedGroupUIDs):
self.log.debug("Enqueuing group refresh for {u}", u=groupUID)
- yield GroupRefreshWork.reschedule(txn, 0, groupUid=groupUID)
+ yield GroupRefreshWork.reschedule(txn, 0, groupUID=groupUID)
@inlineCallbacks
@@ -335,9 +335,9 @@
)
else:
yield GroupDelegateChangesWork.reschedule(
- txn, 0, delegatorUid=delegatorUID,
- readDelegateUid=readDelegateUID,
- writeDelegateUid=writeDelegateUID
+ txn, 0, delegatorUID=delegatorUID,
+ readDelegateUID=readDelegateUID,
+ writeDelegateUID=writeDelegateUID
)
if removed:
for delegatorUID in removed:
@@ -351,8 +351,8 @@
)
else:
yield GroupDelegateChangesWork.reschedule(
- txn, 0, delegatorUid=delegatorUID,
- readDelegateUid="", writeDelegateUid=""
+ txn, 0, delegatorUID=delegatorUID,
+ readDelegateUID="", writeDelegateUID=""
)
@@ -367,26 +367,20 @@
readDelegateGroupID = writeDelegateGroupID = None
if readDelegateUID:
- (
- readDelegateGroupID, _ignore_name, _ignore_hash,
- _ignore_modified, _ignore_extant
- ) = (
- yield txn.groupByUID(readDelegateUID)
- )
- if readDelegateGroupID is None:
+ readDelegateGroup = yield txn.groupByUID(readDelegateUID)
+ if readDelegateGroup is None:
# The group record does not actually exist
readDelegateUID = None
+ else:
+ readDelegateGroupID = readDelegateGroup.groupID
if writeDelegateUID:
- (
- writeDelegateGroupID, _ignore_name, _ignore_hash,
- _ignore_modified, _ignore_extant
- ) = (
- yield txn.groupByUID(writeDelegateUID)
- )
- if writeDelegateGroupID is None:
+ writeDelegateGroup = yield txn.groupByUID(writeDelegateUID)
+ if writeDelegateGroup is None:
# The group record does not actually exist
writeDelegateUID = None
+ else:
+ writeDelegateGroupID = writeDelegateGroup.groupID
yield txn.assignExternalDelegates(
delegatorUID, readDelegateGroupID, writeDelegateGroupID,
@@ -411,45 +405,36 @@
else:
self.log.debug("Got group record: {u}", u=record.uid)
- (
- groupID, cachedName, cachedMembershipHash, _ignore_modified,
- cachedExtant
- ) = yield txn.groupByUID(
- groupUID,
- create=(record is not None)
- )
+ group = yield txn.groupByUID(groupUID, create=(record is not None))
- if groupID:
- membershipChanged, addedUIDs, removedUIDs = yield txn.refreshGroup(
- groupUID, record, groupID,
- cachedName, cachedMembershipHash, cachedExtant
- )
+ if group:
+ membershipChanged, addedUIDs, removedUIDs = yield txn.refreshGroup(group, record)
if membershipChanged:
self.log.info(
"Membership changed for group {uid} {name}:\n\tadded {added}\n\tremoved {removed}",
- uid=groupUID,
- name=cachedName,
+ uid=group.groupUID,
+ name=group.name,
added=",".join(addedUIDs),
removed=",".join(removedUIDs),
)
# Send cache change notifications
if self.cacheNotifier is not None:
- self.cacheNotifier.changed(groupUID)
+ self.cacheNotifier.changed(group.groupUID)
for uid in itertools.chain(addedUIDs, removedUIDs):
self.cacheNotifier.changed(uid)
# Notifier other store APIs of changes
- wpsAttendee = yield self.scheduleGroupAttendeeReconciliations(txn, groupID)
- wpsShareee = yield self.scheduleGroupShareeReconciliations(txn, groupID)
+ wpsAttendee = yield self.scheduleGroupAttendeeReconciliations(txn, group.groupID)
+ wpsShareee = yield self.scheduleGroupShareeReconciliations(txn, group.groupID)
returnValue(wpsAttendee + wpsShareee)
else:
self.log.debug(
"No membership change for group {uid} {name}",
- uid=groupUID,
- name=cachedName
+ uid=group.groupUID,
+ name=group.name
)
returnValue(tuple())
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_delegates.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_delegates.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_delegates.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -19,6 +19,8 @@
"""
from txdav.common.datastore.sql import CommonStoreTransaction
+from txdav.common.datastore.sql_directory import DelegateRecord, \
+ DelegateGroupsRecord
from txdav.who.delegates import Delegates, RecordType as DelegateRecordType
from txdav.who.groups import GroupCacher
from twext.who.idirectory import RecordType
@@ -211,12 +213,9 @@
yield self.directory.recordWithShortName(RecordType.user, name)
)
newSet.add(record.uid)
- (
- groupID, name, _ignore_membershipHash, _ignore_modified,
- _ignore_extant
- ) = (yield txn.groupByUID(group1.uid))
+ group = yield txn.groupByUID(group1.uid)
_ignore_added, _ignore_removed = (
- yield self.groupCacher.synchronizeMembers(txn, groupID, newSet)
+ yield self.groupCacher.synchronizeMembers(txn, group.groupID, newSet)
)
delegates = (yield Delegates.delegatesOf(txn, delegator, True, expanded=True))
self.assertEquals(
@@ -261,15 +260,14 @@
yield txn.commit()
txn = self.store.newTransaction(label="test_noDuplication")
- results = (
- yield txn._selectDelegatesQuery.on(
- txn,
- delegator=delegator.uid.encode("utf-8"),
- readWrite=1
+ results = yield DelegateRecord.query(
+ txn,
+ (DelegateRecord.delegator == delegator.uid.encode("utf-8")).And(
+ DelegateRecord.readWrite == 1
)
)
yield txn.commit()
- self.assertEquals([["__sagen1__"]], results)
+ self.assertEquals(["__sagen1__", ], [record.delegate for record in results])
# Delegate groups:
group1 = yield self.directory.recordWithUID(u"__top_group_1__")
@@ -283,15 +281,13 @@
yield txn.commit()
txn = self.store.newTransaction(label="test_noDuplication")
- results = (
- yield txn._selectDelegateGroupsQuery.on(
- txn,
- delegator=delegator.uid.encode("utf-8"),
- readWrite=1
- )
+ results = yield DelegateGroupsRecord.delegateGroups(
+ txn,
+ delegator.uid,
+ True,
)
yield txn.commit()
- self.assertEquals([["__top_group_1__"]], results)
+ self.assertEquals(["__top_group_1__", ], [record.groupUID for record in results])
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -871,14 +871,11 @@
# finally, simulate an event that has become old
self.patch(CalendarDirectoryRecordMixin, "expandedMembers", unpatchedExpandedMembers)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modDate,
- _ignore_extant
- ) = yield self.transactionUnderTest().groupByUID("group01")
+ group = yield self.transactionUnderTest().groupByUID("group01")
ga = schema.GROUP_ATTENDEE
yield Insert({
ga.RESOURCE_ID: cobj._resourceID,
- ga.GROUP_ID: groupID,
+ ga.GROUP_ID: group.groupID,
ga.MEMBERSHIP_HASH: (-1),
}).on(self.transactionUnderTest())
wps = yield groupCacher.refreshGroup(self.transactionUnderTest(), "group01")
@@ -1033,14 +1030,11 @@
# finally, simulate an event that has become old
self.patch(CalendarDirectoryRecordMixin, "expandedMembers", unpatchedExpandedMembers)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modDate,
- _ignore_extant
- ) = yield self.transactionUnderTest().groupByUID("group01")
+ group = yield self.transactionUnderTest().groupByUID("group01")
ga = schema.GROUP_ATTENDEE
yield Insert({
ga.RESOURCE_ID: cobj._resourceID,
- ga.GROUP_ID: groupID,
+ ga.GROUP_ID: group.groupID,
ga.MEMBERSHIP_HASH: (-1),
}).on(self.transactionUnderTest())
wps = yield groupCacher.refreshGroup(self.transactionUnderTest(), "group01")
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_groups.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_groups.py 2015-02-16 20:33:55 UTC (rev 14416)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_groups.py 2015-02-16 20:56:21 UTC (rev 14417)
@@ -67,27 +67,24 @@
record = yield self.directory.recordWithUID(u"__top_group_1__")
yield self.groupCacher.refreshGroup(txn, record.uid)
- (
- groupID, _ignore_name, membershipHash, _ignore_modified,
- extant
- ) = (yield txn.groupByUID(record.uid))
+ group = (yield txn.groupByUID(record.uid))
- self.assertEquals(extant, True)
- self.assertEquals(membershipHash, "553eb54e3bbb26582198ee04541dbee4")
+ self.assertEquals(group.extant, True)
+ self.assertEquals(group.membershipHash, "553eb54e3bbb26582198ee04541dbee4")
- groupUID, name, membershipHash, extant = (yield txn.groupByID(groupID))
- self.assertEquals(groupUID, record.uid)
- self.assertEquals(name, u"Top Group 1")
- self.assertEquals(membershipHash, "553eb54e3bbb26582198ee04541dbee4")
- self.assertEquals(extant, True)
+ group = yield txn.groupByID(group.groupID)
+ self.assertEquals(group.groupUID, record.uid)
+ self.assertEquals(group.name, u"Top Group 1")
+ self.assertEquals(group.membershipHash, "553eb54e3bbb26582198ee04541dbee4")
+ self.assertEquals(group.extant, True)
- members = (yield txn.groupMemberUIDs(groupID))
+ members = (yield txn.groupMemberUIDs(group.groupID))
self.assertEquals(
set([u'__cdaboo1__', u'__glyph1__', u'__sagen1__', u'__wsanchez1__']),
members
)
- records = (yield self.groupCacher.cachedMembers(txn, groupID))
+ records = (yield self.groupCacher.cachedMembers(txn, group.groupID))
self.assertEquals(
set([r.uid for r in records]),
set([u'__cdaboo1__', u'__glyph1__', u'__sagen1__', u'__wsanchez1__'])
@@ -116,10 +113,7 @@
# Refresh the group so it's assigned a group_id
uid = u"__top_group_1__"
yield self.groupCacher.refreshGroup(txn, uid)
- (
- groupID, name, _ignore_membershipHash, _ignore_modified,
- _ignore_extant
- ) = yield txn.groupByUID(uid)
+ group = yield txn.groupByUID(uid)
# Remove two members, and add one member
newSet = set()
@@ -133,12 +127,12 @@
newSet.add(record.uid)
added, removed = (
yield self.groupCacher.synchronizeMembers(
- txn, groupID, newSet
+ txn, group.groupID, newSet
)
)
self.assertEquals(added, set(["__dre1__", ]))
self.assertEquals(removed, set(["__glyph1__", "__sagen1__", ]))
- records = (yield self.groupCacher.cachedMembers(txn, groupID))
+ records = (yield self.groupCacher.cachedMembers(txn, group.groupID))
self.assertEquals(
set([r.shortNames[0] for r in records]),
set(["wsanchez1", "cdaboo1", "dre1"])
@@ -146,11 +140,11 @@
# Remove all members
added, removed = (
- yield self.groupCacher.synchronizeMembers(txn, groupID, set())
+ yield self.groupCacher.synchronizeMembers(txn, group.groupID, set())
)
self.assertEquals(added, set())
self.assertEquals(removed, set(["__wsanchez1__", "__cdaboo1__", "__dre1__", ]))
- records = (yield self.groupCacher.cachedMembers(txn, groupID))
+ records = (yield self.groupCacher.cachedMembers(txn, group.groupID))
self.assertEquals(len(records), 0)
yield txn.commit()
@@ -168,12 +162,12 @@
uid = u"__top_group_1__"
hash = "553eb54e3bbb26582198ee04541dbee4"
yield self.groupCacher.refreshGroup(txn, uid)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- _ignore_extant
- ) = yield txn.groupByUID(uid)
- results = yield txn.groupByID(groupID)
- self.assertEquals((uid, u"Top Group 1", hash, True), results)
+ group = yield txn.groupByUID(uid)
+ group = yield txn.groupByID(group.groupID)
+ self.assertEqual(group.groupUID, uid)
+ self.assertEqual(group.name, u"Top Group 1")
+ self.assertEqual(group.membershipHash, hash)
+ self.assertEqual(group.extant, True)
yield txn.commit()
@@ -683,31 +677,25 @@
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
- (
- _ignore_groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- extant
- ) = (yield txn.groupByUID(uid))
+ group = yield txn.groupByUID(uid)
yield txn.commit()
- self.assertTrue(extant)
+ self.assertTrue(group.extant)
# Remove the group
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- extant
- ) = (yield txn.groupByUID(uid))
+ group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = False
- self.assertFalse(extant)
+ self.assertFalse(group.extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
- members = yield txn.groupMemberUIDs(groupID)
+ members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(members, set())
@@ -732,18 +720,15 @@
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- extant
- ) = (yield txn.groupByUID(uid))
+ group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = True
- self.assertTrue(extant)
+ self.assertTrue(group.extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
- members = yield txn.groupMemberUIDs(groupID)
+ members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(len(members), 100 if uid == u"testgroup" else 0)
@@ -760,27 +745,27 @@
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
- groupID = (yield txn.groupByUID(uid, create=False))[0]
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
- self.assertNotEqual(groupID, None)
+ self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
- groupID = (yield txn.groupByUID(uid, create=False))[0]
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
- self.assertEqual(groupID, None)
+ self.assertEqual(group, None)
# delegate groups not deleted
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
- groupID = (yield txn.groupByUID(uid))[0]
- yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=groupID, readWrite=True)
+ group = yield txn.groupByUID(uid)
+ yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True)
yield txn.commit()
- self.assertNotEqual(groupID, None)
+ self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
@@ -788,21 +773,21 @@
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
- groupID = (yield txn.groupByUID(uid, create=False))[0]
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
- self.assertNotEqual(groupID, None)
+ self.assertNotEqual(group, None)
# delegate group is deleted. unused group is deleted
txn = store.newTransaction()
- testGroupID = (yield txn.groupByUID(u"testgroup", create=False))[0]
- yield txn.removeDelegateGroup(delegator=u"sagen", delegateGroupID=testGroupID, readWrite=True)
- testGroupID = (yield txn.groupByUID(u"testgroup", create=False))[0]
- emptyGroupID = (yield txn.groupByUID(u"emptygroup", create=False))[0]
+ testGroup = yield txn.groupByUID(u"testgroup", create=False)
+ yield txn.removeDelegateGroup(delegator=u"sagen", delegateGroupID=testGroup.groupID, readWrite=True)
+ testGroup = yield txn.groupByUID(u"testgroup", create=False)
+ emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
- self.assertNotEqual(testGroupID, None)
- self.assertNotEqual(emptyGroupID, None)
+ self.assertNotEqual(testGroup, None)
+ self.assertNotEqual(emptyGroup, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
@@ -810,12 +795,12 @@
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
- testGroupID = (yield txn.groupByUID(u"testgroup", create=False))[0]
- emptyGroupID = (yield txn.groupByUID(u"emptygroup", create=False))[0]
+ testGroup = yield txn.groupByUID(u"testgroup", create=False)
+ emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
- self.assertEqual(testGroupID, None)
- self.assertNotEqual(emptyGroupID, None)
+ self.assertEqual(testGroup, None)
+ self.assertNotEqual(emptyGroup, None)
@inlineCallbacks
@@ -831,42 +816,33 @@
config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds
txn = store.newTransaction()
- groupID = (yield txn.groupByUID(uid))[0]
- yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=groupID, readWrite=True)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- extant
- ) = yield txn.groupByUID(uid, create=False)
+ group = yield txn.groupByUID(uid)
+ yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True)
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
- self.assertTrue(extant)
- self.assertNotEqual(groupID, None)
+ self.assertNotEqual(group, None)
+ self.assertTrue(group.extant)
# Remove the group, still cached
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.update(txn)
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- extant
- ) = yield txn.groupByUID(uid, create=False)
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
- (
- groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
- extant
- ) = yield txn.groupByUID(uid, create=False)
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
- self.assertNotEqual(groupID, None)
- self.assertFalse(extant)
+ self.assertNotEqual(group, None)
+ self.assertFalse(group.extant)
# delete the group
config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0"
txn = store.newTransaction()
yield self.groupCacher.update(txn)
- groupID = (yield txn.groupByUID(uid, create=False))[0]
+ group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
- self.assertEqual(groupID, None)
+ self.assertEqual(group, None)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150216/1fd9944f/attachment-0001.html>
More information about the calendarserver-changes
mailing list