[CalendarServer-changes] [6270] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Sep 10 11:12:03 PDT 2010
Revision: 6270
http://trac.macosforge.org/projects/calendarserver/changeset/6270
Author: sagen at apple.com
Date: 2010-09-10 11:12:02 -0700 (Fri, 10 Sep 2010)
Log Message:
-----------
Adds a bulk createRecords method to make migration of resources/locations fast.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/resources.py
CalendarServer/trunk/calendarserver/tools/test/test_resources.py
CalendarServer/trunk/twistedcaldav/directory/aggregate.py
CalendarServer/trunk/twistedcaldav/directory/directory.py
CalendarServer/trunk/twistedcaldav/directory/xmlfile.py
CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Modified: CalendarServer/trunk/calendarserver/tools/resources.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/resources.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/calendarserver/tools/resources.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -27,6 +27,7 @@
from twisted.internet.defer import inlineCallbacks
from twisted.python.util import switchUID
from twistedcaldav.config import config, ConfigurationError
+from twistedcaldav.directory import augment
from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
from twistedcaldav.directory.directory import DirectoryService, DirectoryError
from twistedcaldav.directory.xmlfile import XMLDirectoryService
@@ -190,6 +191,9 @@
def migrateResources(sourceService, destService, autoSchedules=None,
queryMethod=queryForType, verbose=False):
+ directoryRecords = []
+ augmentRecords = []
+
for recordTypeOD, recordType in (
(dsattributes.kDSStdRecordTypeResources, DirectoryService.recordType_resources),
(dsattributes.kDSStdRecordTypePlaces, DirectoryService.recordType_locations),
@@ -205,14 +209,30 @@
if record is None:
if verbose:
print "Migrating %s (%s)" % (fullName, recordType)
+
if autoSchedules is not None:
autoSchedule = autoSchedules.get(guid, 1)
else:
autoSchedule = True
- yield updateRecord(True, destService, recordType,
- guid=guid, shortNames=[recordName], fullName=fullName,
- autoSchedule=autoSchedule)
+ augmentRecord = (yield augment.AugmentService.getAugmentRecord(guid, recordType))
+ augmentRecord.autoSchedule = autoSchedule
+ augmentRecords.append(augmentRecord)
+ directoryRecords.append(
+ (recordType,
+ {
+ "guid" : guid,
+ "shortNames" : [recordName],
+ "fullName" : fullName,
+ }
+ )
+ )
+ destService.createRecords(directoryRecords)
+
+ (yield augment.AugmentService.addAugmentRecords(augmentRecords))
+
+
+
if __name__ == "__main__":
main()
Modified: CalendarServer/trunk/calendarserver/tools/test/test_resources.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_resources.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/calendarserver/tools/test/test_resources.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -43,12 +43,13 @@
def recordWithGUID(self, guid):
return None
- def createRecord(self, recordType, guid=None, shortNames=None,
- fullName=None):
- record = StubDirectoryRecord(recordType, guid=guid,
- shortNames=shortNames, fullName=fullName)
- self.records[guid] = record
- return record
+ def createRecords(self, data):
+ for recordType, recordData in data:
+ guid = recordData["guid"]
+ record = StubDirectoryRecord(recordType, guid=guid,
+ shortNames=recordData['shortNames'],
+ fullName=recordData['fullName'])
+ self.records[guid] = record
def updateRecord(self, recordType, guid=None, shortNames=None,
fullName=None):
Modified: CalendarServer/trunk/twistedcaldav/directory/aggregate.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/aggregate.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/twistedcaldav/directory/aggregate.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -236,6 +236,10 @@
service = self.serviceForRecordType(recordType)
return service.destroyRecord(recordType, guid=guid)
+ def createRecords(self, data):
+ service = self.serviceForRecordType(recordType)
+ return service.createRecords(data)
+
def setRealm(self, realmName):
"""
Set a new realm name for this and nested services
Modified: CalendarServer/trunk/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/directory.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/twistedcaldav/directory/directory.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -326,6 +326,11 @@
"""
raise NotImplementedError("Subclass must implement destroyRecord")
+ def createRecords(self, data):
+ """
+ Create directory records in bulk
+ """
+ raise NotImplementedError("Subclass must implement createRecords")
class DirectoryRecord(LoggingMixIn):
implements(IDirectoryRecord)
Modified: CalendarServer/trunk/twistedcaldav/directory/xmlfile.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/xmlfile.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/twistedcaldav/directory/xmlfile.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -305,8 +305,7 @@
indent(element)
- # TODO: make this robust:
- ET.ElementTree(element).write(self.xmlFile.path)
+ self.xmlFile.setContent(ET.tostring(element))
# Reload
self._initCaches() # nuke local cache
@@ -324,7 +323,6 @@
to elementtree elements, a new element is added for the new record,
and the document is serialized to disk.
"""
-
if guid is None:
guid = str(uuid4())
@@ -424,6 +422,52 @@
return self.recordWithGUID(guid)
+ def createRecords(self, data):
+ """
+ Create records in bulk
+ """
+
+ # Make sure latest XML records are read in
+ self._lastCheck = 0
+ accounts = self._accounts()
+
+ knownGUIDs = { }
+ knownShortNames = { }
+
+ accountsElement = ET.Element("accounts", realm=self.realmName)
+ for recType in self.recordTypes():
+ for xmlPrincipal in accounts[recType].itervalues():
+ self._addElement(accountsElement, xmlPrincipal)
+ knownGUIDs[xmlPrincipal.guid] = 1
+ for shortName in xmlPrincipal.shortNames:
+ knownShortNames[shortName] = 1
+
+ for recordType, recordData in data:
+ guid = recordData["guid"]
+ if guid is None:
+ guid = str(uuid4())
+
+ shortNames = recordData["shortNames"]
+ if not shortNames:
+ shortNames = (guid,)
+
+ if guid in knownGUIDs:
+ raise DirectoryError("Duplicate guid: %s" % (guid,))
+
+ for shortName in shortNames:
+ if shortName in knownShortNames:
+ raise DirectoryError("Duplicate shortName: %s" %
+ (shortName,))
+
+ xmlPrincipal = XMLAccountRecord(recordType)
+ xmlPrincipal.shortNames = shortNames
+ xmlPrincipal.guid = guid
+ xmlPrincipal.fullName = recordData["fullName"]
+ self._addElement(accountsElement, xmlPrincipal)
+
+ self._persistRecords(accountsElement)
+
+
class XMLDirectoryRecord(CachingDirectoryRecord):
"""
XML based implementation implementation of L{IDirectoryRecord}.
Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -304,7 +304,7 @@
}
},
".calendarserver_version" : {
- "@contents" : "1",
+ "@contents" : "2",
},
MailGatewayTokensDatabase.dbFilename : { "@contents" : None },
"%s-journal" % (MailGatewayTokensDatabase.dbFilename,) : { "@contents" : None },
@@ -383,7 +383,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -503,7 +503,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -595,7 +595,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -689,7 +689,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -768,7 +768,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -884,7 +884,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -1004,7 +1004,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -1192,7 +1192,7 @@
after = {
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
NEWPROXYFILE :
{
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2010-09-10 17:30:37 UTC (rev 6269)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2010-09-10 18:12:02 UTC (rev 6270)
@@ -505,7 +505,7 @@
upgradeMethods = [
(1, upgrade_to_1),
- # (2, upgrade_to_2), # This is too slow for large installations
+ (2, upgrade_to_2),
]
@inlineCallbacks
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100910/734b3d48/attachment-0001.html>
More information about the calendarserver-changes
mailing list