<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14449] CalendarServer/branches/users/cdaboo/pod2pod-migration</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14449">14449</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-02-19 13:53:05 -0800 (Thu, 19 Feb 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Support group attendee migration. Fix a bunch of stuff related to other meta-data needed for sync'ing. More use of DAL Record.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationrequirementsstabletxt">CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoretestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavwhogroupspy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavwhotesttest_group_attendeespy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_directorypy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_directory.py</a></li>
<li>CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/</li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtestaccountsgroupAccountsxml">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationrequirementsstabletxt"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/requirements-stable.txt 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -36,7 +36,7 @@
</span><span class="cx"> #pyOpenSSL
</span><span class="cx"> pycrypto==2.6.1
</span><span class="cx">
</span><del>- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/branches/users/cdaboo/pod2pod-migration@14422#egg=twextpy
</del><ins>+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/branches/users/cdaboo/pod2pod-migration@14448#egg=twextpy
</ins><span class="cx"> cffi==0.8.6
</span><span class="cx"> pycparser==2.10
</span><span class="cx"> #twisted
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -64,6 +64,7 @@
</span><span class="cx"> from txdav.caldav.datastore.scheduling.icalsplitter import iCalSplitter
</span><span class="cx"> from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
</span><span class="cx"> from txdav.caldav.datastore.scheduling.utils import uidFromCalendarUserAddress
</span><ins>+from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord
</ins><span class="cx"> from txdav.caldav.datastore.util import AttachmentRetrievalTransport, \
</span><span class="cx"> normalizationLookup
</span><span class="cx"> from txdav.caldav.datastore.util import CalendarObjectBase
</span><span class="lines">@@ -503,40 +504,22 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def copyMetadata(self, other):
</del><ins>+ def copyMetadata(self, other, calendarIDMap):
</ins><span class="cx"> """
</span><span class="cx"> Copy metadata from one L{CalendarObjectResource} to another. This is only
</span><span class="cx"> used during a migration step.
</span><span class="cx"> """
</span><del>- assert self._txn._migrating
</del><span class="cx">
</span><del>- # Simple attributes that can be copied over as-is
</del><ins>+ # Simple attributes that can be copied over as-is, but the calendar id's need to be mapped
</ins><span class="cx"> chm = self._homeMetaDataSchema
</span><del>- values = {
- chm.ALARM_VEVENT_TIMED : other._alarm_vevent_timed,
- chm.ALARM_VEVENT_ALLDAY : other._alarm_vevent_allday,
- chm.ALARM_VTODO_TIMED : other._alarm_vtodo_timed,
- chm.ALARM_VTODO_ALLDAY : other._alarm_vtodo_allday,
- chm.AVAILABILITY : other._availability,
- }
</del><ins>+ values = {}
+ for attr, col in zip(self.metadataAttributes(), self.metadataColumns()):
+ value = getattr(other, attr)
+ if attr in self._componentDefaultAttribute.values():
+ value = calendarIDMap.get(value)
+ setattr(self, attr, value)
+ values[col] = value
</ins><span class="cx">
</span><del>- # Need to map the default collection references from the remote ids to
- # the local ones using names
- remote_calendars = yield other.loadChildren()
- remote_calendars = dict([(calendar.id(), calendar,) for calendar in remote_calendars])
- local_calendars = yield self.loadChildren()
- local_calendars = dict([(calendar.name(), calendar,) for calendar in local_calendars])
-
- for componentType in self._componentDefaultColumn.keys():
- attr_name = self._componentDefaultAttribute[componentType]
- remote_id = getattr(other, attr_name)
- if remote_id is not None:
- remote_calendar = remote_calendars.get(remote_id)
- if remote_calendar is not None:
- remote_id = local_calendars.get(remote_calendar.name())
- setattr(self, attr_name, remote_id)
- values[self._componentDefaultColumn[componentType]] = remote_id
-
</del><span class="cx"> # Update the local data
</span><span class="cx"> yield Update(
</span><span class="cx"> values,
</span><span class="lines">@@ -701,6 +684,27 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def getAllGroupAttendees(self):
+ """
+ Return a list of L{GroupAttendeeRecord},L{GroupRecord} for each group attendee referenced in calendar data
+ owned by this home.
+ """
+
+ results = []
+ calendars = yield self.loadChildren()
+ for calendar in calendars:
+ if not calendar.owned():
+ continue
+ children = yield calendar.objectResources()
+ cobjs = [child.id() for child in children]
+ if cobjs:
+ result = yield GroupAttendeeRecord.groupAttendeesForObjects(self._txn, cobjs)
+ results.extend(result)
+
+ returnValue(results)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def createdHome(self):
</span><span class="cx">
</span><span class="cx"> # Check whether components type must be separate
</span><span class="lines">@@ -2519,19 +2523,14 @@
</span><span class="cx"> @return: a L{dict} with group ids as the key and membership hash as the value
</span><span class="cx"> @rtype: L{dict}
</span><span class="cx"> """
</span><del>- ga = schema.GROUP_ATTENDEE
- rows = yield Select(
- [ga.GROUP_ID, ga.MEMBERSHIP_HASH],
- From=ga,
- Where=ga.RESOURCE_ID == self._resourceID,
- ).on(self._txn)
- returnValue(dict(rows))
</del><ins>+ records = yield GroupAttendeeRecord.querysimple(self._txn, resourceID=self._resourceID)
+ returnValue(dict([(record.groupID, record,) for record in records]))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def updateEventGroupLink(self, groupCUAToAttendeeMemberPropMap=None):
</span><span class="cx"> """
</span><del>- update schema.GROUP_ATTENDEE
</del><ins>+ update group event links
</ins><span class="cx"> """
</span><span class="cx"> if groupCUAToAttendeeMemberPropMap is None:
</span><span class="cx"> if hasattr(self, "_groupCUAToAttendeeMemberPropMap"):
</span><span class="lines">@@ -2550,37 +2549,25 @@
</span><span class="cx"> groupUID = uidFromCalendarUserAddress(groupCUA)
</span><span class="cx"> group = yield self._txn.groupByUID(groupUID)
</span><span class="cx">
</span><del>- ga = schema.GROUP_ATTENDEE
</del><span class="cx"> if group.groupID in groupIDToMembershipHashMap:
</span><del>- if groupIDToMembershipHashMap[group.groupID] != group.membershipHash:
- yield Update(
- {ga.MEMBERSHIP_HASH: group.membershipHash, },
- Where=(ga.RESOURCE_ID == self._resourceID).And(
- ga.GROUP_ID == group.groupID)
- ).on(self._txn)
</del><ins>+ if groupIDToMembershipHashMap[group.groupID].membershipHash != group.membershipHash:
+ yield groupIDToMembershipHashMap[group.groupID].update(membershipHash=group.membershipHash)
</ins><span class="cx"> changed = True
</span><span class="cx"> del groupIDToMembershipHashMap[group.groupID]
</span><span class="cx"> else:
</span><del>- yield Insert({
- ga.RESOURCE_ID: self._resourceID,
- ga.GROUP_ID: group.groupID,
- ga.MEMBERSHIP_HASH: group.membershipHash,
- }).on(self._txn)
</del><ins>+ yield GroupAttendeeRecord.create(
+ self._txn,
+ resourceID=self._resourceID,
+ groupID=group.groupID,
+ membershipHash=group.membershipHash,
+ )
</ins><span class="cx"> changed = True
</span><span class="cx">
</span><span class="cx"> if groupIDToMembershipHashMap:
</span><del>- groupIDsToRemove = groupIDToMembershipHashMap.keys()
- yield Delete(
- From=ga,
- Where=(ga.RESOURCE_ID == self._resourceID).And(
- ga.GROUP_ID.In(
- Parameter(
- "groupIDsToRemove",
- len(groupIDsToRemove)
- )
- )
- )
- ).on(self._txn, groupIDsToRemove=groupIDsToRemove)
</del><ins>+ yield GroupAttendeeRecord.deletesome(
+ self._txn,
+ GroupAttendeeRecord.groupID.In(groupIDToMembershipHashMap.keys()),
+ )
</ins><span class="cx"> changed = True
</span><span class="cx">
</span><span class="cx"> returnValue(changed)
</span><span class="lines">@@ -2641,11 +2628,7 @@
</span><span class="cx"> del self._groupCUAToAttendeeMemberPropMap
</span><span class="cx"> else:
</span><span class="cx"> # delete existing group rows
</span><del>- ga = schema.GROUP_ATTENDEE
- yield Delete(
- From=ga,
- Where=ga.RESOURCE_ID == self._resourceID,
- ).on(txn)
</del><ins>+ yield GroupAttendeeRecord.deletesimple(self._txn, resourceID=self._resourceID)
</ins><span class="cx">
</span><span class="cx"> returnValue(isOldEventWithGroupAttendees)
</span><span class="cx">
</span><span class="lines">@@ -2691,13 +2674,11 @@
</span><span class="cx"> # remove group link to ensure update (update to unknown hash would work too)
</span><span class="cx"> # FIXME: its possible that more than one group id gets updated during this single work item, so we
</span><span class="cx"> # need to make sure that ALL the group_id's are removed by this query.
</span><del>- ga = schema.GROUP_ATTENDEE
- yield Delete(
- From=ga,
- Where=(ga.RESOURCE_ID == self._resourceID).And(
- ga.GROUP_ID == groupID
- )
- ).on(self._txn)
</del><ins>+ yield GroupAttendeeRecord.deletesimple(
+ self._txn,
+ resourceID=self._resourceID,
+ groupID=groupID,
+ )
</ins><span class="cx">
</span><span class="cx"> # update group attendee in remaining component
</span><span class="cx"> component = yield self.componentForUser()
</span><span class="lines">@@ -3807,8 +3788,6 @@
</span><span class="cx"> Copy metadata from one L{CalendarObjectResource} to another. This is only
</span><span class="cx"> used during a migration step.
</span><span class="cx"> """
</span><del>- assert self._txn._migrating
-
</del><span class="cx"> co = self._objectSchema
</span><span class="cx"> values = {
</span><span class="cx"> co.ATTACHMENTS_MODE : other._attachment,
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_directorypy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_directory.py (0 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_directory.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_directory.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -0,0 +1,67 @@
</span><ins>+# -*- test-case-name: twext.enterprise.dal.test.test_record -*-
+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.record import SerializableRecord, fromTable
+from twext.enterprise.dal.syntax import Select, Parameter
+from twext.python.log import Logger
+from twisted.internet.defer import inlineCallbacks, returnValue
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.sql_directory import GroupsRecord
+
+log = Logger()
+
+"""
+Classes and methods that relate to directory objects in the SQL store. e.g.,
+delegates, groups etc
+"""
+
+class GroupAttendeeRecord(SerializableRecord, fromTable(schema.GROUP_ATTENDEE)):
+ """
+ @DynamicAttrs
+ L{Record} for L{schema.GROUP_ATTENDEE}.
+ """
+
+ @classmethod
+ @inlineCallbacks
+ def groupAttendeesForObjects(cls, txn, cobjs):
+ """
+ Get delegator/group pairs for each of the specified calendar objects.
+ """
+
+ # Do a join to get what we need
+ rows = yield Select(
+ list(GroupAttendeeRecord.table) + list(GroupsRecord.table),
+ From=GroupAttendeeRecord.table.join(GroupsRecord.table, GroupAttendeeRecord.groupID == GroupsRecord.groupID),
+ Where=(GroupAttendeeRecord.resourceID.In(Parameter("cobjs", len(cobjs))))
+ ).on(txn, cobjs=cobjs)
+
+ results = []
+ groupAttendeeNames = [GroupAttendeeRecord.__colmap__[column] for column in list(GroupAttendeeRecord.table)]
+ groupsNames = [GroupsRecord.__colmap__[column] for column in list(GroupsRecord.table)]
+ split_point = len(groupAttendeeNames)
+ for row in rows:
+ groupAttendeeRow = row[:split_point]
+ groupAttendeeRecord = GroupAttendeeRecord()
+ groupAttendeeRecord._attributesFromRow(zip(groupAttendeeNames, groupAttendeeRow))
+ groupAttendeeRecord.transaction = txn
+ groupsRow = row[split_point:]
+ groupsRecord = GroupsRecord()
+ groupsRecord._attributesFromRow(zip(groupsNames, groupsRow))
+ groupsRecord.transaction = txn
+ results.append((groupAttendeeRecord, groupsRecord,))
+
+ returnValue(results)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -14,6 +14,8 @@
</span><span class="cx"> # See the License for the specific language governing permissions and
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><ins>+from txdav.common.datastore.sql_directory import GroupsRecord
+from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord
</ins><span class="cx"> """
</span><span class="cx"> SQL backend for CalDAV storage when resources are external.
</span><span class="cx"> """
</span><span class="lines">@@ -113,6 +115,17 @@
</span><span class="cx"> raise AssertionError("CommonHomeExternal: not supported")
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def getAllGroupAttendees(self):
+ """
+ Return a list of L{GroupAttendeeRecord},L{GroupRecord} for each group attendee referenced in calendar data
+ owned by this home.
+ """
+
+ raw_results = yield self._txn.store().conduit.send_home_get_all_group_attendees(self)
+ returnValue([(GroupAttendeeRecord.deserialize(item[0]), GroupsRecord.deserialize(item[1]),) for item in raw_results])
+
+
</ins><span class="cx"> def createdHome(self):
</span><span class="cx"> """
</span><span class="cx"> No children - make this a no-op.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -173,18 +173,18 @@
</span><span class="cx"> # TODO: Re-write attachment URIs - not sure if we need this as reverse proxy may take care of it
</span><span class="cx"> pass
</span><span class="cx">
</span><ins>+ # TODO: group attendee reconcile
+ yield self.groupAttendeeReconcile()
+
+ # TODO: delegates reconcile
+ yield self.delegateReconcile()
+
</ins><span class="cx"> # TODO: shared collections reconcile
</span><span class="cx"> pass
</span><span class="cx">
</span><del>- # TODO: group attendee reconcile
- pass
-
</del><span class="cx"> # TODO: group sharee reconcile
</span><span class="cx"> pass
</span><span class="cx">
</span><del>- # TODO: delegates reconcile
- yield self.delegateReconcile()
-
</del><span class="cx"> # TODO: notifications
</span><span class="cx"> pass
</span><span class="cx">
</span><span class="lines">@@ -259,8 +259,13 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> remote_home = yield self._remoteHome(txn=txn)
</span><ins>+ yield remote_home.readMetaData()
+
+ calendars = yield CalendarMigrationRecord.querysimple(txn, calendarHomeResourceID=self.homeId)
+ calendarIDMap = dict((item.remoteResourceID, item.localResourceID) for item in calendars)
+
</ins><span class="cx"> local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</span><del>- yield local_home.copyMetadata(remote_home)
</del><ins>+ yield local_home.copyMetadata(remote_home, calendarIDMap)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -832,3 +837,56 @@
</span><span class="cx"> remote_records = yield txn.dumpExternalDelegatesExternal(self.record)
</span><span class="cx"> for record in remote_records:
</span><span class="cx"> yield record.insert(txn)
</span><ins>+
+
+ @inlineCallbacks
+ def groupAttendeeReconcile(self):
+ """
+ Sync the remote group attendee links to the local store.
+ """
+
+ # Get remote data and local mapping information
+ remote_group_attendees, objectIDMap = yield self.groupAttendeeData()
+
+ # Map each result to a local resource (in batches)
+ number_of_links = len(remote_group_attendees)
+ while remote_group_attendees:
+ yield self.groupAttendeeProcess(remote_group_attendees[:50], objectIDMap)
+ remote_group_attendees = remote_group_attendees[50:]
+
+ returnValue(number_of_links)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def groupAttendeeData(self, txn):
+ """
+ Sync the remote group attendee links to the local store.
+ """
+ remote_home = yield self._remoteHome(txn)
+ remote_group_attendees = yield remote_home.getAllGroupAttendees()
+
+ # Get all remote->local object maps
+ records = yield CalendarObjectMigrationRecord.querysimple(
+ txn, calendarHomeResourceID=self.homeId
+ )
+ objectIDMap = dict([(record.remoteResourceID, record.localResourceID) for record in records])
+
+ returnValue((remote_group_attendees, objectIDMap,))
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def groupAttendeeProcess(self, txn, results, objectIDMap):
+ """
+ Sync the remote group attendee links to the local store.
+ """
+ # Map each result to a local resource
+ for groupAttendee, group in results:
+ local_group = yield txn.groupByUID(group.groupUID)
+ groupAttendee.groupID = local_group.groupID
+ try:
+ groupAttendee.resourceID = objectIDMap[groupAttendee.resourceID]
+ except KeyError:
+ continue
+ yield groupAttendee.insert(txn)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtestaccountsgroupAccountsxml"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml (0 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -0,0 +1,140 @@
</span><ins>+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+Copyright (c) 2006-2015 Apple Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ -->
+
+<!DOCTYPE accounts SYSTEM "accounts.dtd">
+
+<directory realm="Test Realm">
+ <record type="user">
+ <short-name>user01</short-name>
+ <uid>user01</uid>
+ <guid>10000000-0000-0000-0000-000000000001</guid>
+ <password>user01</password>
+ <full-name>User 01</full-name>
+ <email>user01@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user02</short-name>
+ <uid>user02</uid>
+ <guid>10000000-0000-0000-0000-000000000002</guid>
+ <password>user02</password>
+ <full-name>User 02</full-name>
+ <email>user02@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user03</short-name>
+ <uid>user03</uid>
+ <guid>10000000-0000-0000-0000-000000000003</guid>
+ <password>user03</password>
+ <full-name>User 03</full-name>
+ <email>user03@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user04</short-name>
+ <uid>user04</uid>
+ <guid>10000000-0000-0000-0000-000000000004</guid>
+ <password>user04</password>
+ <full-name>User 04</full-name>
+ <email>user04@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user05</short-name>
+ <uid>user05</uid>
+ <guid>10000000-0000-0000-0000-000000000005</guid>
+ <password>user05</password>
+ <full-name>User 05</full-name>
+ <email>user05@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user06</short-name>
+ <uid>user06</uid>
+ <guid>10000000-0000-0000-0000-000000000006</guid>
+ <password>user06</password>
+ <full-name>User 06</full-name>
+ <email>user06@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user07</short-name>
+ <uid>user07</uid>
+ <guid>10000000-0000-0000-0000-000000000007</guid>
+ <password>user07</password>
+ <full-name>User 07</full-name>
+ <email>user07@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user08</short-name>
+ <uid>user08</uid>
+ <guid>10000000-0000-0000-0000-000000000008</guid>
+ <password>user08</password>
+ <full-name>User 08</full-name>
+ <email>user08@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user09</short-name>
+ <uid>user09</uid>
+ <guid>10000000-0000-0000-0000-000000000009</guid>
+ <password>user09</password>
+ <full-name>User 09</full-name>
+ <email>user09@example.com</email>
+ </record>
+ <record type="user">
+ <short-name>user10</short-name>
+ <uid>user10</uid>
+ <guid>10000000-0000-0000-0000-000000000010</guid>
+ <password>user10</password>
+ <full-name>User 10</full-name>
+ <email>user10@example.com</email>
+ </record>
+ <record type="group">
+ <short-name>group01</short-name>
+ <uid>group01</uid>
+ <guid>20000000-0000-0000-0000-000000000001</guid>
+ <full-name>Group 01</full-name>
+ <email>group01@example.com</email>
+ <member-uid>user01</member-uid>
+ </record>
+ <record type="group">
+ <short-name>group02</short-name>
+ <uid>group02</uid>
+ <guid>20000000-0000-0000-0000-000000000002</guid>
+ <full-name>Group 02</full-name>
+ <email>group02@example.com</email>
+ <member-uid>user06</member-uid>
+ <member-uid>user07</member-uid>
+ <member-uid>user08</member-uid>
+ </record>
+ <record type="group">
+ <short-name>group03</short-name>
+ <uid>group03</uid>
+ <guid>20000000-0000-0000-0000-000000000003</guid>
+ <full-name>Group 03</full-name>
+ <email>group03@example.com</email>
+ <member-uid>user07</member-uid>
+ <member-uid>user08</member-uid>
+ <member-uid>user09</member-uid>
+ </record>
+ <record type="group">
+ <short-name>group04</short-name>
+ <uid>group04</uid>
+ <guid>20000000-0000-0000-0000-000000000004</guid>
+ <full-name>Group 04</full-name>
+ <email>group04@example.com</email>
+ <member-uid>group02</member-uid>
+ <member-uid>group03</member-uid>
+ <member-uid>user10</member-uid>
+ </record>
+</directory>
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -16,15 +16,21 @@
</span><span class="cx">
</span><span class="cx"> from pycalendar.datetime import DateTime
</span><span class="cx"> from twext.enterprise.dal.syntax import Select
</span><del>-from twisted.internet.defer import inlineCallbacks
</del><ins>+from twext.enterprise.jobqueue import JobItem
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python.filepath import FilePath
+from twistedcaldav.config import config
</ins><span class="cx"> from twistedcaldav.ical import Component, normalize_iCalStr
</span><span class="cx"> from txdav.caldav.datastore.sql import ManagedAttachment
</span><span class="cx"> from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
</span><del>-from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord
</del><ins>+from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord, \
+ AttachmentMigrationRecord
</ins><span class="cx"> from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
</span><span class="cx"> from txdav.common.datastore.sql_directory import DelegateRecord, \
</span><span class="cx"> ExternalDelegateGroupsRecord, DelegateGroupsRecord
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema
</span><ins>+from txdav.common.datastore.test.util import populateCalendarsFrom
</ins><span class="cx"> from txdav.who.delegates import Delegates
</span><span class="cx"> from txweb2.http_headers import MimeType
</span><span class="cx"> from txweb2.stream import MemoryStream
</span><span class="lines">@@ -117,6 +123,7 @@
</span><span class="cx"> END:VCALENDAR
</span><span class="cx"> """.replace("\n", "\r\n").format(**nowYear)
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def test_remote_home(self):
</span><span class="cx"> """
</span><span class="lines">@@ -181,6 +188,115 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def test_home_metadata(self):
+ """
+ Test that L{syncCalendarHomeMetaData} sync home metadata correctly.
+ """
+
+ alarm_event_timed = """BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:alarm_event_timed
+TRIGGER:-PT10M
+END:VALARM
+"""
+ alarm_event_allday = """BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:alarm_event_allday
+TRIGGER:-PT10M
+END:VALARM
+"""
+ alarm_todo_timed = """BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:alarm_todo_timed
+TRIGGER:-PT10M
+END:VALARM
+"""
+ alarm_todo_allday = """BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:alarm_todo_allday
+TRIGGER:-PT10M
+END:VALARM
+"""
+ availability = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Example Inc.//Example Calendar//EN
+BEGIN:VAVAILABILITY
+UID:20061005T133225Z-00001-availability@example.com
+DTSTART:20060101T000000Z
+DTEND:20060108T000000Z
+DTSTAMP:20061005T133225Z
+ORGANIZER:mailto:bernard@example.com
+BEGIN:AVAILABLE
+UID:20061005T133225Z-00001-A-availability@example.com
+DTSTART:20060102T090000Z
+DTEND:20060102T120000Z
+DTSTAMP:20061005T133225Z
+RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
+SUMMARY:Weekdays from 9:00 to 12:00
+END:AVAILABLE
+END:VAVAILABILITY
+END:VCALENDAR
+"""
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ events0 = yield home0.createChildWithName("events")
+ yield home0.setDefaultCalendar(events0, "VEVENT")
+ yield home0.setDefaultAlarm(alarm_event_timed, True, True)
+ yield home0.setDefaultAlarm(alarm_event_allday, True, False)
+ yield home0.setDefaultAlarm(alarm_todo_timed, False, True)
+ yield home0.setDefaultAlarm(alarm_todo_allday, False, False)
+ yield home0.setAvailability(Component.fromString(availability))
+ yield self.commitTransaction(0)
+
+ # Trigger sync
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.sync()
+
+ # Home is present with correct metadata
+ home1 = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ self.assertTrue(home1 is not None)
+ calendar1 = yield home1.childWithName("calendar")
+ events1 = yield home1.childWithName("events")
+ tasks1 = yield home1.childWithName("tasks")
+ self.assertFalse(home1.isDefaultCalendar(calendar1))
+ self.assertTrue(home1.isDefaultCalendar(events1))
+ self.assertTrue(home1.isDefaultCalendar(tasks1))
+ self.assertEqual(home1.getDefaultAlarm(True, True), alarm_event_timed)
+ self.assertEqual(home1.getDefaultAlarm(True, False), alarm_event_allday)
+ self.assertEqual(home1.getDefaultAlarm(False, True), alarm_todo_timed)
+ self.assertEqual(home1.getDefaultAlarm(False, False), alarm_todo_allday)
+ self.assertEqual(normalize_iCalStr(home1.getAvailability()), normalize_iCalStr(availability))
+ yield self.commitTransaction(1)
+
+ # Make some changes
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ yield home0.setDefaultCalendar(calendar0, "VEVENT")
+ yield home0.setDefaultAlarm(None, True, True)
+ yield home0.setDefaultAlarm(None, False, True)
+ yield self.commitTransaction(0)
+
+ # Trigger sync again
+ yield syncer.sync()
+
+ # Home is present with correct metadata
+ home1 = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ self.assertTrue(home1 is not None)
+ calendar1 = yield home1.childWithName("calendar")
+ events1 = yield home1.childWithName("events")
+ tasks1 = yield home1.childWithName("tasks")
+ self.assertTrue(home1.isDefaultCalendar(calendar1))
+ self.assertFalse(home1.isDefaultCalendar(events1))
+ self.assertTrue(home1.isDefaultCalendar(tasks1))
+ self.assertEqual(home1.getDefaultAlarm(True, True), None)
+ self.assertEqual(home1.getDefaultAlarm(True, False), alarm_event_allday)
+ self.assertEqual(home1.getDefaultAlarm(False, True), None)
+ self.assertEqual(home1.getDefaultAlarm(False, False), alarm_todo_allday)
+ self.assertEqual(normalize_iCalStr(home1.getAvailability()), normalize_iCalStr(availability))
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def test_get_calendar_sync_list(self):
</span><span class="cx"> """
</span><span class="cx"> Test that L{getCalendarSyncList} returns the correct results.
</span><span class="lines">@@ -475,6 +591,16 @@
</span><span class="cx"> self.assertEqual(len(local_sync_state), 1)
</span><span class="cx"> self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _mapLocalIDToRemote(remote_id):
+ records = yield AttachmentMigrationRecord.all(self.theTransactionUnderTest(1))
+ yield self.commitTransaction(1)
+ for record in records:
+ if record.remoteResourceID == remote_id:
+ returnValue(record.localResourceID)
+ else:
+ returnValue(None)
+
</ins><span class="cx"> # Sync attachments
</span><span class="cx"> changed, removed = yield syncer.syncAttachments()
</span><span class="cx"> self.assertEqual(changed, set())
</span><span class="lines">@@ -515,7 +641,7 @@
</span><span class="cx">
</span><span class="cx"> # Sync attachments
</span><span class="cx"> changed, removed = yield syncer.syncAttachments()
</span><del>- self.assertEqual(changed, set((id0_1,)))
</del><ins>+ self.assertEqual(changed, set(((yield _mapLocalIDToRemote(id0_1)),)))
</ins><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><span class="lines">@@ -534,7 +660,7 @@
</span><span class="cx">
</span><span class="cx"> # Sync attachments
</span><span class="cx"> changed, removed = yield syncer.syncAttachments()
</span><del>- self.assertEqual(changed, set((id0_2,)))
</del><ins>+ self.assertEqual(changed, set(((yield _mapLocalIDToRemote(id0_2)),)))
</ins><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><span class="lines">@@ -555,7 +681,7 @@
</span><span class="cx">
</span><span class="cx"> # Sync attachments
</span><span class="cx"> changed, removed = yield syncer.syncAttachments()
</span><del>- self.assertEqual(changed, set((id0_1_changed,)))
</del><ins>+ self.assertEqual(changed, set(((yield _mapLocalIDToRemote(id0_1_changed)),)))
</ins><span class="cx"> self.assertEqual(removed, set((id0_1,)))
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><span class="lines">@@ -596,7 +722,7 @@
</span><span class="cx">
</span><span class="cx"> # Sync attachments
</span><span class="cx"> changed, removed = yield syncer.syncAttachments()
</span><del>- self.assertEqual(changed, set((id0_1_changed_again,)))
</del><ins>+ self.assertEqual(changed, set(((yield _mapLocalIDToRemote(id0_1_changed_again)),)))
</ins><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><span class="lines">@@ -661,7 +787,18 @@
</span><span class="cx">
</span><span class="cx"> # Sync attachments
</span><span class="cx"> changed, removed = yield syncer.syncAttachments()
</span><del>- self.assertEqual(changed, set((id0_1, id0_2,)))
</del><ins>+
+ @inlineCallbacks
+ def _mapLocalIDToRemote(remote_id):
+ records = yield AttachmentMigrationRecord.all(self.theTransactionUnderTest(1))
+ yield self.commitTransaction(1)
+ for record in records:
+ if record.remoteResourceID == remote_id:
+ returnValue(record.localResourceID)
+ else:
+ returnValue(None)
+
+ self.assertEqual(changed, set(((yield _mapLocalIDToRemote(id0_1)), (yield _mapLocalIDToRemote(id0_2)),)))
</ins><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Link attachments
</span><span class="lines">@@ -787,3 +924,131 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> yield self.commitTransaction(1)
</span><ins>+
+
+
+class TestGroupAttendeeSync(MultiStoreConduitTest):
+ """
+ GroupAttendeeReconciliation tests
+ """
+
+ now = {"now1": DateTime.getToday().getYear() + 1}
+
+ groupdata1 = """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VEVENT
+DTSTAMP:20051222T205953Z
+CREATED:20060101T150000Z
+DTSTART:{now1:04d}0101T100000Z
+DURATION:PT1H
+SUMMARY:event 1
+UID:event1@ninevah.local
+END:VEVENT
+END:VCALENDAR""".format(**now)
+
+ groupdata2 = """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VEVENT
+DTSTAMP:20051222T205953Z
+CREATED:20060101T150000Z
+DTSTART:{now1:04d}0101T100000Z
+DURATION:PT1H
+SUMMARY:event 2
+UID:event2@ninevah.local
+ORGANIZER:mailto:user01@example.com
+ATTENDEE:mailto:user01@example.com
+ATTENDEE:mailto:group02@example.com
+END:VEVENT
+END:VCALENDAR""".format(**now)
+
+ groupdata3 = """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VEVENT
+DTSTAMP:20051222T205953Z
+CREATED:20060101T150000Z
+DTSTART:{now1:04d}0101T100000Z
+DURATION:PT1H
+SUMMARY:event 3
+UID:event3@ninevah.local
+ORGANIZER:mailto:user01@example.com
+ATTENDEE:mailto:user01@example.com
+ATTENDEE:mailto:group04@example.com
+END:VEVENT
+END:VCALENDAR""".format(**now)
+
+ @inlineCallbacks
+ def setUp(self):
+ self.accounts = FilePath(__file__).sibling("accounts").child("groupAccounts.xml")
+ yield super(TestGroupAttendeeSync, self).setUp()
+ yield self.populate()
+
+
+ def configure(self):
+ super(TestGroupAttendeeSync, self).configure()
+ config.GroupAttendees.Enabled = True
+ config.GroupAttendees.ReconciliationDelaySeconds = 0
+ config.GroupAttendees.AutoUpdateSecondsFromNow = 0
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateCalendarsFrom(self.requirements, self.theStoreUnderTest(0))
+
+ requirements = {
+ "user01" : None,
+ "user02" : None,
+ "user06" : None,
+ "user07" : None,
+ "user08" : None,
+ "user09" : None,
+ "user10" : None,
+
+ }
+
+ @inlineCallbacks
+ def test_group_attendees(self):
+ """
+ Test that L{groupAttendeeReconcile} links groups to the associated calendar object.
+ """
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.groupdata1))
+ yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.groupdata2))
+ yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.groupdata3))
+ yield self.commitTransaction(0)
+
+ yield JobItem.waitEmpty(self.theStoreUnderTest(0).newTransaction, reactor, 60.0)
+
+ # Trigger sync
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.sync()
+
+ # Link groups
+ len_links = yield syncer.groupAttendeeReconcile()
+ self.assertEqual(len_links, 2)
+
+ # Local calendar exists
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is not None)
+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "2.ics", "3.ics",)))
+
+ object2 = yield calendar1.objectResourceWithName("2.ics")
+ record = (yield object2.groupEventLinks()).values()[0]
+ group02 = yield self.theTransactionUnderTest(1).groupByUID(u"group02")
+ self.assertEqual(record.groupID, group02.groupID)
+ self.assertEqual(record.membershipHash, group02.membershipHash)
+
+ object3 = yield calendar1.objectResourceWithName("3.ics")
+ record = (yield object3.groupEventLinks()).values()[0]
+ group04 = yield self.theTransactionUnderTest(1).groupByUID(u"group04")
+ self.assertEqual(record.groupID, group04.groupID)
+ self.assertEqual(record.membershipHash, group04.membershipHash)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -127,9 +127,19 @@
</span><span class="cx"> "matchtotal": matchtotal,
</span><span class="cx"> })
</span><span class="cx">
</span><ins>+
+ @staticmethod
+ def _to_serialize_pair_list(value):
+ """
+ Convert the value to the external (JSON-based) representation.
+ """
+ return [[a.serialize(), b.serialize(), ] for a, b in value]
+
</ins><span class="cx"> # These are the actions on store objects we need to expose via the conduit api
</span><span class="cx">
</span><span class="cx"> # Calls on L{CommonHome} objects
</span><ins>+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_metadata", "serialize", classMethod=False)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_get_all_group_attendees", "getAllGroupAttendees", classMethod=False, transform_recv_result=StoreAPIConduitMixin._to_serialize_pair_list)
</ins><span class="cx">
</span><span class="cx"> # Calls on L{CommonHomeChild} objects
</span><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_listobjects", "listObjects", classMethod=True)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -128,6 +128,7 @@
</span><span class="cx"> theStoreBuilders = []
</span><span class="cx"> theStores = []
</span><span class="cx"> activeTransactions = []
</span><ins>+ accounts = None
</ins><span class="cx">
</span><span class="cx"> def __init__(self, methodName='runTest'):
</span><span class="cx"> txweb2.dav.test.util.TestCase.__init__(self, methodName)
</span><span class="lines">@@ -153,13 +154,17 @@
</span><span class="cx"> if i == 0:
</span><span class="cx"> yield self.buildStoreAndDirectory(
</span><span class="cx"> serversDB=serversDB,
</span><del>- storeBuilder=self.theStoreBuilders[i]
</del><ins>+ storeBuilder=self.theStoreBuilders[i],
+ accounts=self.accounts,
</ins><span class="cx"> )
</span><span class="cx"> self.theStores[i] = self.store
</span><span class="cx"> else:
</span><span class="cx"> self.theStores[i] = yield self.buildStore(self.theStoreBuilders[i])
</span><span class="cx"> directory = buildTestDirectory(
</span><del>- self.theStores[i], self.mktemp(), serversDB=serversDB
</del><ins>+ self.theStores[i],
+ self.mktemp(),
+ serversDB=serversDB,
+ accounts=self.accounts,
</ins><span class="cx"> )
</span><span class="cx"> self.theStores[i].setDirectoryService(directory)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -2191,6 +2191,24 @@
</span><span class="cx"> setattr(self, attr, value)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def serialize(self):
+ """
+ Create a dictionary mapping metadata attributes so this object can be sent over a cross-pod call
+ and reconstituted at the other end. Note that the other end may have a different schema so
+ the attributes may not match exactly and will need to be processed accordingly.
+ """
+ return dict([(attr[1:], getattr(self, attr, None)) for attr in self.metadataAttributes()])
+
+
+ def deserialize(self, mapping):
+ """
+ Given a mapping generated by L{serialize}, convert the values to attributes on this object.
+ """
+
+ for attr in self.metadataAttributes():
+ setattr(self, attr, mapping.get(attr[1:]))
+
+
</ins><span class="cx"> @classmethod
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def listHomes(cls, txn):
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -53,6 +53,15 @@
</span><span class="cx"> raise AssertionError("CommonHomeExternal: not supported")
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def readMetaData(self):
+ """
+ Read the home metadata from remote home and save as attributes on this object.
+ """
+ mapping = yield self._txn.store().conduit.send_home_metadata(self)
+ self.deserialize(mapping)
+
+
</ins><span class="cx"> def external(self):
</span><span class="cx"> """
</span><span class="cx"> Is this an external home.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -927,7 +927,7 @@
</span><span class="cx"> DELEGATOR varchar(255) not null,
</span><span class="cx"> GROUP_ID integer not null references GROUPS on delete cascade,
</span><span class="cx"> READ_WRITE integer not null, -- 1 = ReadWrite, 0 = ReadOnly
</span><del>- IS_EXTERNAL integer not null, -- 1 = ReadWrite, 0 = ReadOnly
</del><ins>+ IS_EXTERNAL integer not null, -- 1 = External, 0 = Internal
</ins><span class="cx">
</span><span class="cx"> primary key (DELEGATOR, READ_WRITE, GROUP_ID)
</span><span class="cx"> );
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoretestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -30,7 +30,7 @@
</span><span class="cx">
</span><span class="cx"> from pycalendar.datetime import DateTime
</span><span class="cx">
</span><del>-from random import Random
</del><ins>+from random import Random, randint
</ins><span class="cx">
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twext.python.filepath import CachingFilePath as FilePath
</span><span class="lines">@@ -290,6 +290,16 @@
</span><span class="cx"> yield cleanupTxn.execSQL("delete from " + table, [])
</span><span class="cx"> except:
</span><span class="cx"> log.failure("delete table {table} failed", table=table)
</span><ins>+
+ # Change the starting values of sequences to random values
+ for sequence in schema.model.sequences: #@UndefinedVariable
+ try:
+ curval = (yield cleanupTxn.execSQL("select nextval('{}')".format(sequence.name), []))[0][0]
+ yield cleanupTxn.execSQL("select setval('{}', {})".format(sequence.name, curval + randint(1, 10000)), [])
+ except:
+ log.failure("setval sequence '{}' failed", sequence=sequence.name)
+ yield cleanupTxn.execSQL("update CALENDARSERVER set VALUE = '1' where NAME = 'MIN-VALID-REVISION'", [])
+
</ins><span class="cx"> yield cleanupTxn.commit()
</span><span class="cx">
</span><span class="cx"> # Deal with memcached items that must be cleared
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavwhogroupspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/groups.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -20,13 +20,15 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.record import fromTable
</span><del>-from twext.enterprise.dal.syntax import Delete, Select, Parameter
</del><ins>+from twext.enterprise.dal.syntax import Select
</ins><span class="cx"> from twext.enterprise.jobqueue import AggregatedWorkItem, RegeneratingWorkItem
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue, succeed, \
</span><span class="cx"> DeferredList
</span><span class="cx"> from twistedcaldav.config import config
</span><span class="cx"> from txdav.caldav.datastore.sql import CalendarStoreFeatures
</span><ins>+from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord
+from txdav.common.datastore.sql_directory import GroupsRecord
</ins><span class="cx"> from txdav.common.datastore.sql_tables import schema, _BIND_MODE_OWN
</span><span class="cx"> import datetime
</span><span class="cx"> import itertools
</span><span class="lines">@@ -268,33 +270,28 @@
</span><span class="cx"> # "Groups to refresh: {g}", g=groupUIDs
</span><span class="cx"> # )
</span><span class="cx">
</span><del>- gr = schema.GROUPS
</del><span class="cx"> if config.AutomaticPurging.Enabled and groupUIDs:
</span><span class="cx"> # remove unused groups and groups that have not been seen in a while
</span><span class="cx"> dateLimit = (
</span><span class="cx"> datetime.datetime.utcnow() -
</span><span class="cx"> datetime.timedelta(seconds=float(config.AutomaticPurging.GroupPurgeIntervalSeconds))
</span><span class="cx"> )
</span><del>- rows = yield Delete(
- From=gr,
- Where=(
- (gr.EXTANT == 0).And(gr.MODIFIED < dateLimit)
</del><ins>+ rows = yield GroupsRecord.deletesome(
+ txn,
+ (
+ (GroupsRecord.extant == 0).And(GroupsRecord.modified < dateLimit)
</ins><span class="cx"> ).Or(
</span><del>- gr.GROUP_UID.NotIn(
- Parameter("groupUIDs", len(groupUIDs))
- )
- ) if groupUIDs else None,
- Return=[gr.GROUP_UID]
- ).on(txn, groupUIDs=groupUIDs)
</del><ins>+ GroupsRecord.groupUID.NotIn(groupUIDs)
+ ),
+ returnCols=GroupsRecord.groupUID,
+ )
</ins><span class="cx"> else:
</span><span class="cx"> # remove unused groups
</span><del>- rows = yield Delete(
- From=gr,
- Where=gr.GROUP_UID.NotIn(
- Parameter("groupUIDs", len(groupUIDs))
- ) if groupUIDs else None,
- Return=[gr.GROUP_UID]
- ).on(txn, groupUIDs=groupUIDs)
</del><ins>+ rows = yield GroupsRecord.deletesome(
+ txn,
+ GroupsRecord.groupUID.NotIn(groupUIDs) if groupUIDs else None,
+ returnCols=GroupsRecord.groupUID,
+ )
</ins><span class="cx"> deletedGroupUIDs = [row[0] for row in rows]
</span><span class="cx"> if deletedGroupUIDs:
</span><span class="cx"> self.log.debug("Deleted old or unused groups {d}", d=deletedGroupUIDs)
</span><span class="lines">@@ -465,19 +462,15 @@
</span><span class="cx"> work items for them.
</span><span class="cx"> returns: WorkProposal
</span><span class="cx"> """
</span><del>- ga = schema.GROUP_ATTENDEE
- rows = yield Select(
- [ga.RESOURCE_ID, ],
- From=ga,
- Where=ga.GROUP_ID == groupID,
- ).on(txn)
</del><span class="cx">
</span><ins>+ records = yield GroupAttendeeRecord.querysimple(txn, groupID=groupID)
+
</ins><span class="cx"> wps = []
</span><del>- for [eventID] in rows:
</del><ins>+ for record in records:
</ins><span class="cx"> wp = yield GroupAttendeeReconciliationWork.reschedule(
</span><span class="cx"> txn,
</span><span class="cx"> seconds=float(config.GroupAttendees.ReconciliationDelaySeconds),
</span><del>- resourceID=eventID,
</del><ins>+ resourceID=record.resourceID,
</ins><span class="cx"> groupID=groupID,
</span><span class="cx"> )
</span><span class="cx"> wps.append(wp)
</span><span class="lines">@@ -531,20 +524,15 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> # Get groupUIDs for all group attendees
</span><del>- ga = schema.GROUP_ATTENDEE
- gr = schema.GROUPS
- rows = yield Select(
- [gr.GROUP_UID],
- From=gr,
- Where=gr.GROUP_ID.In(
- Select(
- [ga.GROUP_ID],
- From=ga,
- Distinct=True
- )
- )
- ).on(txn)
- attendeeGroupUIDs = frozenset([row[0] for row in rows])
</del><ins>+ groups = yield GroupsRecord.query(
+ txn,
+ GroupsRecord.groupID.In(GroupAttendeeRecord.queryExpr(
+ expr=None,
+ attributes=(GroupAttendeeRecord.groupID,),
+ distinct=True,
+ ))
+ )
+ attendeeGroupUIDs = frozenset([group.groupUID for group in groups])
</ins><span class="cx"> self.log.info(
</span><span class="cx"> "There are {count} group attendees", count=len(attendeeGroupUIDs)
</span><span class="cx"> )
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavwhotesttest_group_attendeespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py (14448 => 14449)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py 2015-02-19 21:40:25 UTC (rev 14448)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_attendees.py 2015-02-19 21:53:05 UTC (rev 14449)
</span><span class="lines">@@ -18,7 +18,6 @@
</span><span class="cx"> group attendee tests
</span><span class="cx"> """
</span><span class="cx">
</span><del>-from twext.enterprise.dal.syntax import Insert
</del><span class="cx"> from twext.enterprise.jobqueue import JobItem
</span><span class="cx"> from twext.python.filepath import CachingFilePath as FilePath
</span><span class="cx"> from twext.who.directory import DirectoryService
</span><span class="lines">@@ -27,8 +26,8 @@
</span><span class="cx"> from twisted.trial import unittest
</span><span class="cx"> from twistedcaldav.config import config
</span><span class="cx"> from twistedcaldav.ical import Component, normalize_iCalStr
</span><ins>+from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord
</ins><span class="cx"> from txdav.caldav.datastore.test.util import populateCalendarsFrom, CommonCommonTests
</span><del>-from txdav.common.datastore.sql_tables import schema
</del><span class="cx"> from txdav.who.directory import CalendarDirectoryRecordMixin
</span><span class="cx"> from txdav.who.groups import GroupCacher
</span><span class="cx"> import os
</span><span class="lines">@@ -872,12 +871,12 @@
</span><span class="cx"> self.patch(CalendarDirectoryRecordMixin, "expandedMembers", unpatchedExpandedMembers)
</span><span class="cx">
</span><span class="cx"> group = yield self.transactionUnderTest().groupByUID("group01")
</span><del>- ga = schema.GROUP_ATTENDEE
- yield Insert({
- ga.RESOURCE_ID: cobj._resourceID,
- ga.GROUP_ID: group.groupID,
- ga.MEMBERSHIP_HASH: (-1),
- }).on(self.transactionUnderTest())
</del><ins>+ yield GroupAttendeeRecord.create(
+ self.transactionUnderTest(),
+ resourceID=cobj._resourceID,
+ groupID=group.groupID,
+ membershipHash=-1,
+ )
</ins><span class="cx"> wps = yield groupCacher.refreshGroup(self.transactionUnderTest(), "group01")
</span><span class="cx"> self.assertEqual(len(wps), 1)
</span><span class="cx"> yield self.commit()
</span><span class="lines">@@ -1031,12 +1030,12 @@
</span><span class="cx"> self.patch(CalendarDirectoryRecordMixin, "expandedMembers", unpatchedExpandedMembers)
</span><span class="cx">
</span><span class="cx"> group = yield self.transactionUnderTest().groupByUID("group01")
</span><del>- ga = schema.GROUP_ATTENDEE
- yield Insert({
- ga.RESOURCE_ID: cobj._resourceID,
- ga.GROUP_ID: group.groupID,
- ga.MEMBERSHIP_HASH: (-1),
- }).on(self.transactionUnderTest())
</del><ins>+ yield GroupAttendeeRecord.create(
+ self.transactionUnderTest(),
+ resourceID=cobj._resourceID,
+ groupID=group.groupID,
+ membershipHash=-1,
+ )
</ins><span class="cx"> wps = yield groupCacher.refreshGroup(self.transactionUnderTest(), "group01")
</span><span class="cx"> self.assertEqual(len(wps), 1)
</span><span class="cx"> yield self.commit()
</span></span></pre>
</div>
</div>
</body>
</html>