[CalendarServer-changes] [3762] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Feb 27 15:52:43 PST 2009
Revision: 3762
http://trac.macosforge.org/projects/calendarserver/changeset/3762
Author: sagen at apple.com
Date: 2009-02-27 15:52:42 -0800 (Fri, 27 Feb 2009)
Log Message:
-----------
Landing the migration/upgrade branch
Existing calendar data will be moved to the proper calendars/__uids__/xx/yy/guid
directory, with illegal quoting fixed, and calendar user addresses normalized.
Etags and Ctags are updated as necessary.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tools/util.py
CalendarServer/trunk/twistedcaldav/directory/directory.py
CalendarServer/trunk/twistedcaldav/directory/test/accounts.xml
CalendarServer/trunk/twistedcaldav/directory/test/test_sqldb.py
CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py
CalendarServer/trunk/twistedcaldav/ical.py
CalendarServer/trunk/twistedcaldav/resource.py
CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
CalendarServer/trunk/twistedcaldav/test/util.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Added Paths:
-----------
CalendarServer/trunk/calendarserver/tools/migrate.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -73,7 +73,7 @@
from twistedcaldav.static import TimezoneServiceFile
from twistedcaldav.mail import IMIPInboxResource
from twistedcaldav.timezones import TimezoneCache
-from twistedcaldav.upgrade import UpgradeTheServer
+from twistedcaldav.upgrade import upgradeData
from twistedcaldav.pdmonster import PDClientAddressWrapper
from twistedcaldav import memcachepool
from twistedcaldav.notify import installNotificationClient
@@ -327,8 +327,6 @@
def makeService(self, options):
- # Now do any on disk upgrades we might need.
- UpgradeTheServer.doUpgrade()
serviceMethod = getattr(self, "makeService_%s" % (config.ProcessType,), None)
@@ -339,6 +337,16 @@
% (config.ProcessType,)
)
else:
+
+ if config.ProcessType in ('Combined', 'Single'):
+
+ # Process localization string files
+ processLocalizationFiles(config.Localization)
+
+ # Now do any on disk upgrades we might need.
+ upgradeData(config)
+
+
service = serviceMethod(options)
#
Added: CalendarServer/trunk/calendarserver/tools/migrate.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/migrate.py (rev 0)
+++ CalendarServer/trunk/calendarserver/tools/migrate.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+This tool migrates existing calendar data from any previous calendar server
+version to the current version.
+
+This tool requires access to the calendar server's configuration and
+data storage; it does not operate by talking to the server via the
+network.
+"""
+
+import os
+import sys
+from getopt import getopt, GetoptError
+from os.path import dirname, abspath
+
+from twistedcaldav.upgrade import upgradeData
+from calendarserver.tools.util import loadConfig
+
+def usage(e=None):
+ if e:
+ print e
+ print ""
+
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print "Migrate calendar data to current version"
+ print __doc__
+ print "options:"
+ print " -h --help: print this help and exit"
+ print " -f --config: Specify caldavd.plist configuration path"
+
+ if e:
+ sys.exit(64)
+ else:
+ sys.exit(0)
+
+def main():
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "hf:", [
+ "config=",
+ "help",
+ ],
+ )
+ except GetoptError, e:
+ usage(e)
+
+ configFileName = None
+
+
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ usage()
+
+ elif opt in ("-f", "--config"):
+ configFileName = arg
+
+ if args:
+ usage("Too many arguments: %s" % (" ".join(args),))
+
+ config = loadConfig(configFileName)
+
+ profiling = False
+
+ if profiling:
+ import cProfile
+ cProfile.runctx("upgradeData(c)", globals(), {"c" : config}, "/tmp/upgrade.prof")
+ else:
+ upgradeData(config)
+
+if __name__ == "__main__":
+ main()
Property changes on: CalendarServer/trunk/calendarserver/tools/migrate.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: CalendarServer/trunk/calendarserver/tools/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/util.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/calendarserver/tools/util.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -44,7 +44,7 @@
BaseDirectoryService = namedClass(config.DirectoryService.type)
class MyDirectoryService (BaseDirectoryService):
- def principalCollection(self):
+ def getPrincipalCollection(self):
if not hasattr(self, "_principalCollection"):
#
# Instantiating a CalendarHomeProvisioningResource with a directory
@@ -60,12 +60,22 @@
return self._principalCollection
+ def setPrincipalCollection(self, coll):
+ # See principal.py line 237: self.directory.principalCollection = self
+ pass
+
+ principalCollection = property(getPrincipalCollection, setPrincipalCollection)
+
def calendarHomeForShortName(self, recordType, shortName):
- principal = self.principalCollection().principalForShortName(recordType, shortName)
+ principal = self.principalCollection.principalForShortName(recordType, shortName)
if principal:
return principal.calendarHome()
return None
+ def principalForCalendarUserAddress(self, cua):
+ return self.principalCollection.principalForCalendarUserAddress(cua)
+
+
return MyDirectoryService(**config.DirectoryService.params)
class DummyDirectoryService (DirectoryService):
Modified: CalendarServer/trunk/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/directory.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/directory/directory.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -149,14 +149,7 @@
return self.recordWithGUID(guid)
elif address.startswith("mailto:"):
email = address[7:]
- result = self.recordWithEmailAddress(email)
- if result:
- return result
-
- for record in self.allRecords():
- if address in record.calendarUserAddresses:
- return record
-
+ return self.recordWithEmailAddress(email)
return None
def recordWithEmailAddress(self, email):
Modified: CalendarServer/trunk/twistedcaldav/directory/test/accounts.xml
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/accounts.xml 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/directory/test/accounts.xml 2009-02-27 23:52:42 UTC (rev 3762)
@@ -30,6 +30,7 @@
<guid>6423F94A-6B76-4A3A-815B-D52CFD77935D</guid>
<password>zehcnasw</password>
<name>Wilfredo Sanchez</name>
+ <email-address>wsanchez at example.com</email-address>
<cuaddr>mailto:wsanchez at example.com</cuaddr>
</user>
<user>
@@ -37,6 +38,7 @@
<guid>5A985493-EE2C-4665-94CF-4DFEA3A89500</guid>
<password>oobadc</password>
<name>Cyrus Daboo</name>
+ <email-address>cdaboo at example.com</email-address>
<cuaddr>mailto:cdaboo at example.com</cuaddr>
</user>
<user>
@@ -44,6 +46,7 @@
<guid>8B4288F6-CC82-491D-8EF9-642EF4F3E7D0</guid>
<password>yorcel</password>
<name>Chris Lecroy</name>
+ <email-address>lecroy at example.com</email-address>
<cuaddr>mailto:lecroy at example.com</cuaddr>
</user>
<user>
@@ -51,6 +54,7 @@
<guid>5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1</guid>
<password>dierd</password>
<name>David Reid</name>
+ <email-address>dreid at example.com</email-address>
<cuaddr>mailto:dreid at example.com</cuaddr>
</user>
<user repeat="2">
@@ -60,6 +64,7 @@
</user>
<group>
<uid>managers</uid>
+ <guid>9FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1</guid>
<password>managers</password>
<name>Managers</name>
<members>
@@ -143,6 +148,7 @@
<uid>mercury</uid>
<password>mercury</password>
<name>Mecury Seven</name>
+ <email-address>mercury at example.com</email-address>
<cuaddr>mailto:mercury at example.com</cuaddr>
<proxies>
<member type="groups">left_coast</member>
@@ -152,6 +158,7 @@
<uid>gemini</uid>
<password>gemini</password>
<name>Gemini Twelve</name>
+ <email-address>gemini at example.com</email-address>
<cuaddr>mailto:gemini at example.com</cuaddr>
<auto-schedule/>
<proxies>
@@ -162,6 +169,7 @@
<uid>apollo</uid>
<password>apollo</password>
<name>Apollo Eleven</name>
+ <email-address>apollo at example.com</email-address>
<cuaddr>mailto:apollo at example.com</cuaddr>
<proxies>
<member type="groups">both_coasts</member>
@@ -171,6 +179,7 @@
<uid>orion</uid>
<password>orion</password>
<name>Orion</name>
+ <email-address>orion at example.com</email-address>
<cuaddr>mailto:orion at example.com</cuaddr>
<proxies>
<member type="groups">recursive1_coasts</member>
@@ -180,12 +189,14 @@
<uid>transporter</uid>
<password>transporter</password>
<name>Mass Transporter</name>
+ <email-address>transporter at example.com</email-address>
<cuaddr>mailto:transporter at example.com</cuaddr>
</resource>
<resource>
<uid>ftlcpu</uid>
<password>ftlcpu</password>
<name>Faster-Than-Light Microprocessor</name>
+ <email-address>ftlcpu at example.com</email-address>
<cuaddr>mailto:ftlcpu at example.com</cuaddr>
</resource>
<resource>
@@ -193,6 +204,7 @@
<guid>non_calendar_proxy</guid>
<password>non_calendar_proxy</password>
<name>Non-calendar proxy</name>
+ <email-address>non_calendar_proxy at example.com</email-address>
<cuaddr>mailto:non_calendar_proxy at example.com</cuaddr>
<proxies>
<member type="groups">non_calendar_group</member>
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_sqldb.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_sqldb.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_sqldb.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -26,26 +26,26 @@
# FIXME: Add tests for GUID hooey, once we figure out what that means here
-class SQLDB (
- twistedcaldav.directory.test.test_xmlfile.XMLFileBase,
- twistedcaldav.directory.test.util.BasicTestCase,
- twistedcaldav.directory.test.util.DigestTestCase
-):
- """
- Test SQL directory implementation.
- """
- def service(self):
- return SQLDirectoryService(os.getcwd(), self.xmlFile())
-
- def test_verifyCredentials_digest(self):
- super(SQLDB, self).test_verifyCredentials_digest()
- test_verifyCredentials_digest.todo = ""
-
- def test_verifyRealmFromDB(self):
- # Make sure the database has been initialized with the XML file
- self.service()
-
- # Then get an instance without using the XML file
- service = SQLDirectoryService(os.getcwd(), None)
-
- self.assertEquals(service.realmName, "Test")
+# class SQLDB (
+# twistedcaldav.directory.test.test_xmlfile.XMLFileBase,
+# twistedcaldav.directory.test.util.BasicTestCase,
+# twistedcaldav.directory.test.util.DigestTestCase
+# ):
+# """
+# Test SQL directory implementation.
+# """
+# def service(self):
+# return SQLDirectoryService(os.getcwd(), self.xmlFile())
+#
+# def test_verifyCredentials_digest(self):
+# super(SQLDB, self).test_verifyCredentials_digest()
+# test_verifyCredentials_digest.todo = ""
+#
+# def test_verifyRealmFromDB(self):
+# # Make sure the database has been initialized with the XML file
+# self.service()
+#
+# # Then get an instance without using the XML file
+# service = SQLDirectoryService(os.getcwd(), None)
+#
+# self.assertEquals(service.realmName, "Test")
Modified: CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -274,6 +274,7 @@
elif child_name == ELEMENT_EMAIL_ADDRESS:
if child.firstChild is not None:
self.emailAddresses.add(child.firstChild.data.encode("utf-8").lower())
+ self.calendarUserAddresses.add("mailto:%s" % (child.firstChild.data.encode("utf-8"),))
elif child_name == ELEMENT_MEMBERS:
self._parseMembers(child, self.members)
elif child_name == ELEMENT_CUADDR:
Modified: CalendarServer/trunk/twistedcaldav/ical.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/ical.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/ical.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -45,6 +45,7 @@
import cStringIO as StringIO
import datetime
import heapq
+import itertools
log = Logger()
@@ -1808,6 +1809,66 @@
if dataValue.find(dropboxPrefix) != -1:
component.removeProperty(attachment)
+ def normalizeCalendarUserAddresses(self, lookupFunction):
+ """
+ Do the ORGANIZER/ATTENDEE property normalization.
+
+ @param lookupFunction: function returning full name, guid, CUAs for a given CUA
+ @type lookupFunction: L{Function}
+ """
+ for component in self.subcomponents():
+ if component.name() == "VTIMEZONE":
+ continue
+ for prop in itertools.chain(
+ component.properties("ORGANIZER"),
+ component.properties("ATTENDEE")
+ ):
+
+ # Check that we can lookup this calendar user address - if not
+ # we cannot do anything with it
+ cuaddr = normalizeCUAddr(prop.value())
+ name, guid, cuaddrs = lookupFunction(cuaddr)
+ if guid is None:
+ continue
+
+ # Always re-write value to urn:uuid
+ prop.setValue("urn:uuid:%s" % (guid,))
+
+ # Always re-write the CN parameter
+ if name:
+ prop.params()["CN"] = [name,]
+ else:
+ try:
+ del prop.params()["CN"]
+ except KeyError:
+ pass
+
+ # Re-write the X-CALENDARSERVER-EMAIL if its value no longer
+ # matches
+ oldemail = prop.params().get("X-CALENDARSERVER-EMAIL",
+ (None,))[0]
+ if oldemail:
+ oldemail = "mailto:%s" % (oldemail,)
+ if oldemail is None or oldemail not in cuaddrs:
+ if cuaddr.startswith("mailto:") and cuaddr in cuaddrs:
+ email = cuaddr[7:]
+ else:
+ for addr in cuaddrs:
+ if addr.startswith("mailto:"):
+ email = addr[7:]
+ break
+ else:
+ email = None
+
+ if email:
+ prop.params()["X-CALENDARSERVER-EMAIL"] = [email,]
+ else:
+ try:
+ del prop.params()["X-CALENDARSERVER-EMAIL"]
+ except KeyError:
+ pass
+
+
##
# Dates and date-times
##
Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/resource.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -657,64 +657,17 @@
@param ical: calendar object to normalize.
@type ical: L{Component}
"""
-
- def normalizeCalendarUserAddress(prop):
- """
- Do the ORGANIZER/ATTENDEE property normalization.
- @param prop: organizer/attendee property
- @type prop: L{Property}
- """
-
- # Check that we have a principal for this calendar user address - if not we
- # cannot do anything with it
- cuaddr = normalizeCUAddr(prop.value())
+ def lookupFunction(cuaddr):
principal = self.principalForCalendarUserAddress(cuaddr)
if principal is None:
- return
-
- # Always re-write value to urn:uuid
- prop.setValue("urn:uuid:%s" % (principal.record.guid,))
-
- # Always re-write the CN parameter
- if principal.record.fullName:
- prop.params()["CN"] = [principal.record.fullName,]
+ return (None, None, None)
else:
- try:
- del prop.params()["CN"]
- except KeyError:
- pass
+ return (principal.record.fullName, principal.record.guid,
+ principal.record.calendarUserAddresses)
- # Re-write the X-CALENDARSERVER-EMAIL if its value no longer matches
- oldemail = prop.params().get("X-CALENDARSERVER-EMAIL", (None,))[0]
- if oldemail:
- oldemail = "mailto:%s" % (oldemail,)
- if oldemail is None or oldemail not in principal.record.calendarUserAddresses:
- if cuaddr.startswith("mailto:") and cuaddr in principal.record.calendarUserAddresses:
- email = cuaddr[7:]
- else:
- for addr in principal.record.calendarUserAddresses:
- if addr.startswith("mailto:"):
- email = addr[7:]
- break
- else:
- email = None
-
- if email:
- prop.params()["X-CALENDARSERVER-EMAIL"] = [email,]
- else:
- try:
- del prop.params()["X-CALENDARSERVER-EMAIL"]
- except KeyError:
- pass
+ ical.normalizeCalendarUserAddresses(lookupFunction)
- for component in ical.subcomponents():
- if component.name() != "VTIMEZONE":
- for prop in itertools.chain(
- component.properties("ORGANIZER"),
- component.properties("ATTENDEE")
- ):
- normalizeCalendarUserAddress(prop)
def principalForCalendarUserAddress(self, address):
for principalCollection in self.principalCollections():
Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -16,16 +16,32 @@
from twistedcaldav.config import config
from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
-from twistedcaldav.upgrade import UpgradeError
-from twistedcaldav.upgrade import UpgradeTheServer
+from twistedcaldav.upgrade import (
+ UpgradeError, upgradeData, updateFreeBusySet, updateFreeBusyHref
+)
from twistedcaldav.test.util import TestCase
+from calendarserver.tools.util import getDirectory
+from twisted.web2.dav import davxml
-import os
+import os, zlib, cPickle
+
+freeBusyAttr = "WebDAV:{urn:ietf:params:xml:ns:caldav}calendar-free-busy-set"
+cTagAttr = "WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"
+md5Attr = "WebDAV:{http:%2F%2Ftwistedmatrix.com%2Fxml_namespace%2Fdav%2F}getcontentmd5"
+
+
class ProxyDBUpgradeTests(TestCase):
+ def setUpXMLDirectory(self):
+ xmlFile = os.path.join(os.path.dirname(os.path.dirname(__file__)),
+ "directory", "test", "accounts.xml")
+ config.DirectoryService.params.xmlFile = xmlFile
+
+
def setUpInitialStates(self):
-
+ self.setUpXMLDirectory()
+
self.setUpOldDocRoot()
self.setUpOldDocRootWithoutDB()
self.setUpNewDocRoot()
@@ -47,7 +63,6 @@
os.mkdir(os.path.join(principals, "locations"))
os.mkdir(os.path.join(principals, "resources"))
os.mkdir(os.path.join(principals, "sudoers"))
- os.mkdir(os.path.join(self.olddocroot, "calendars"))
proxyDB = CalendarUserProxyDatabase(principals)
proxyDB._db()
@@ -56,6 +71,7 @@
os.path.join(principals, CalendarUserProxyDatabase.dbOldFilename),
)
+
def setUpOldDocRootWithoutDB(self):
# Set up doc root
@@ -104,6 +120,7 @@
config.DocumentRoot = self.olddocroot
config.DataRoot = self.newdataroot
+
# Check pre-conditions
self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
@@ -111,7 +128,7 @@
self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
self.assertFalse(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
- UpgradeTheServer.doUpgrade()
+ upgradeData(config)
# Check post-conditions
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
@@ -133,7 +150,7 @@
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
self.assertFalse(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
- UpgradeTheServer.doUpgrade()
+ upgradeData(config)
# Check post-conditions
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
@@ -153,7 +170,7 @@
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
- UpgradeTheServer.doUpgrade()
+ upgradeData(config)
# Check post-conditions
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
@@ -175,11 +192,726 @@
self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
- self.assertRaises(UpgradeError, UpgradeTheServer.doUpgrade)
+ self.assertRaises(UpgradeError, upgradeData, config)
# Check post-conditions
self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
self.assertTrue(os.path.isdir(os.path.join(config.DocumentRoot, "principals")))
self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
-
\ No newline at end of file
+
+
+ def test_freeBusyUpgrade(self):
+ """
+ Test the updating of calendar-free-busy-set xattrs on inboxes
+ """
+
+ self.setUpInitialStates()
+ directory = getDirectory()
+
+ #
+ # Verify these values require no updating:
+ #
+
+ # Uncompressed XML
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/BB05932F-DCE7-4195-9ED4-0896EAFF3B0B/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ self.assertEquals(updateFreeBusySet(value, directory), None)
+
+ # Zlib compressed XML
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/BB05932F-DCE7-4195-9ED4-0896EAFF3B0B/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ value = zlib.compress(value)
+ self.assertEquals(updateFreeBusySet(value, directory), None)
+
+ # Pickled XML
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/BB05932F-DCE7-4195-9ED4-0896EAFF3B0B/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ doc = davxml.WebDAVDocument.fromString(value)
+ value = cPickle.dumps(doc.root_element)
+ self.assertEquals(updateFreeBusySet(value, directory), None)
+
+
+ #
+ # Verify these values do require updating:
+ #
+
+ expected = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"
+
+ # Uncompressed XML
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ newValue = updateFreeBusySet(value, directory)
+ newValue = zlib.decompress(newValue)
+ self.assertEquals(newValue, expected)
+
+ # Zlib compressed XML
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ value = zlib.compress(value)
+ newValue = updateFreeBusySet(value, directory)
+ newValue = zlib.decompress(newValue)
+ self.assertEquals(newValue, expected)
+
+ # Pickled XML
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ doc = davxml.WebDAVDocument.fromString(value)
+ value = cPickle.dumps(doc.root_element)
+ newValue = updateFreeBusySet(value, directory)
+ newValue = zlib.decompress(newValue)
+ self.assertEquals(newValue, expected)
+
+
+ #
+ # Shortname not in directory, raise an UpgradeError
+ #
+
+ value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/nonexistent/calendar</href>\r\n</calendar-free-busy-set>\r\n"
+ self.assertRaises(UpgradeError, updateFreeBusySet, value, directory)
+
+
+ def test_calendarsUpgradeWithTypes(self):
+ """
+ Verify that calendar homes in the /calendars/<type>/<shortname>/ form
+ are upgraded to /calendars/__uids__/XX/YY/<guid> form
+ """
+
+ self.setUpXMLDirectory()
+ directory = getDirectory()
+
+ before = {
+ "calendars" :
+ {
+ "users" :
+ {
+ "wsanchez" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_before,
+ "@xattrs" :
+ {
+ md5Attr : "12345",
+ },
+ },
+ "@xattrs" :
+ {
+ cTagAttr : "12345",
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ # Pickled XML Doc
+ freeBusyAttr : cPickle.dumps(davxml.WebDAVDocument.fromString("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n").root_element),
+ },
+ },
+ },
+ },
+ "groups" :
+ {
+ "managers" :
+ {
+ "calendar" :
+ {
+ },
+ },
+ },
+ },
+ "principals" :
+ {
+ CalendarUserProxyDatabase.dbOldFilename :
+ {
+ "@contents" : "",
+ }
+ }
+ }
+
+ after = {
+ ".calendarserver_version" :
+ {
+ "@contents" : "1",
+ },
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_after,
+ "@xattrs" :
+ {
+ md5Attr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>967eac8e6cc69b43fb820e8cf438d8e7</getcontentmd5>\r\n"),
+ },
+ },
+ "@xattrs" :
+ {
+ cTagAttr : isValidCTag, # method below
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ },
+ },
+ },
+ },
+ "9F" :
+ {
+ "F6" :
+ {
+ "9FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1" :
+ {
+ "calendar" :
+ {
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+ root = self.createHierarchy(before)
+
+ config.DocumentRoot = root
+ config.DataRoot = root
+
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+ # Ensure that repeating the process doesn't change anything
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+ def test_calendarsUpgradeWithUIDs(self):
+ """
+ Verify that calendar homes in the /calendars/__uids__/<guid>/ form
+ are upgraded to /calendars/__uids__/XX/YY/<guid>/ form
+ """
+
+ self.setUpXMLDirectory()
+ directory = getDirectory()
+
+
+ before = {
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_before,
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ # Plain XML
+ freeBusyAttr : "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n",
+ },
+ },
+ },
+ },
+ },
+ "principals" :
+ {
+ CalendarUserProxyDatabase.dbOldFilename :
+ {
+ "@contents" : "",
+ }
+ }
+ }
+
+ after = {
+ ".calendarserver_version" :
+ {
+ "@contents" : "1",
+ },
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_after,
+ },
+ "@xattrs" :
+ {
+ cTagAttr : isValidCTag, # method below
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+ root = self.createHierarchy(before)
+
+ config.DocumentRoot = root
+ config.DataRoot = root
+
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+ # Ensure that repeating the process doesn't change anything
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+ def test_calendarsUpgradeWithUIDsMultilevel(self):
+ """
+ Verify that calendar homes in the /calendars/__uids__/XX/YY/<guid>/
+ form are upgraded correctly in place
+ """
+
+ self.setUpXMLDirectory()
+ directory = getDirectory()
+
+ before = {
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_before,
+ "@xattrs" :
+ {
+ md5Attr : "12345",
+ },
+ },
+ "@xattrs" :
+ {
+ "ignore" : "extra",
+ cTagAttr : "12345",
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ # Zlib compressed XML
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+ after = {
+ ".calendarserver_version" :
+ {
+ "@contents" : "1",
+ },
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_after,
+ "@xattrs" :
+ {
+ md5Attr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>967eac8e6cc69b43fb820e8cf438d8e7</getcontentmd5>\r\n"),
+ },
+ },
+ "@xattrs" :
+ {
+ "ignore" : "extra",
+ cTagAttr : isValidCTag, # method below
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+ root = self.createHierarchy(before)
+
+ config.DocumentRoot = root
+ config.DataRoot = root
+
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+ # Ensure that repeating the process doesn't change anything
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+
+ def test_calendarsUpgradeWithNoChange(self):
+ """
+ Verify that calendar homes in the /calendars/__uids__/XX/YY/<guid>/
+ form which require no changes are untouched
+ """
+
+ self.setUpXMLDirectory()
+ directory = getDirectory()
+
+ before = {
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_after,
+ "@xattrs" :
+ {
+ md5Attr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>967eac8e6cc69b43fb820e8cf438d8e7</getcontentmd5>\r\n"),
+ },
+ },
+ "@xattrs" :
+ {
+ "ignore" : "extra",
+ cTagAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<getctag xmlns='http://calendarserver.org/ns/'>2009-02-25 14:34:34.703093</getctag>\r\n"),
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ # Zlib compressed XML
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+ after = {
+ ".calendarserver_version" :
+ {
+ "@contents" : "1",
+ },
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_after,
+ "@xattrs" :
+ {
+ md5Attr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>967eac8e6cc69b43fb820e8cf438d8e7</getcontentmd5>\r\n"),
+ },
+ },
+ "@xattrs" :
+ {
+ "ignore" : "extra",
+ cTagAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<getctag xmlns='http://calendarserver.org/ns/'>2009-02-25 14:34:34.703093</getctag>\r\n"),
+ },
+ },
+ "inbox" :
+ {
+ "@xattrs" :
+ {
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+ root = self.createHierarchy(before)
+
+ config.DocumentRoot = root
+ config.DataRoot = root
+
+ upgradeData(config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+
+
+
+ def test_calendarsUpgradeWithError(self):
+ """
+ Verify that a problem with one resource doesn't stop the process, but
+ also doesn't write the new version file
+ """
+
+ self.setUpXMLDirectory()
+ directory = getDirectory()
+
+ before = {
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935E" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_before,
+ },
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C73.ics" :
+ {
+ "@contents" : event02_broken,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+
+ after = {
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935E" :
+ {
+ "calendar" :
+ {
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C72.ics" :
+ {
+ "@contents" : event01_after,
+ },
+ "1E238CA1-3C95-4468-B8CD-C8A399F78C73.ics" :
+ {
+ "@contents" : event02_broken,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ CalendarUserProxyDatabase.dbFilename :
+ {
+ "@contents" : "",
+ }
+ }
+
+
+ root = self.createHierarchy(before)
+
+ config.DocumentRoot = root
+ config.DataRoot = root
+
+ self.assertRaises(UpgradeError, upgradeData, config)
+ self.assertTrue(self.verifyHierarchy(root, after))
+
+
+
+event01_before = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 3.0//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+TZOFFSETTO:-0700
+DTSTART:20070311T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+TZNAME:PDT
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+TZOFFSETTO:-0800
+DTSTART:20071104T020000
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+TZNAME:PST
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+SEQUENCE:2
+TRANSP:OPAQUE
+UID:1E238CA1-3C95-4468-B8CD-C8A399F78C71
+DTSTART;TZID=US/Pacific:20090203T120000
+ORGANIZER;CN="Cyrus":mailto:cdaboo at example.com
+DTSTAMP:20090203T181924Z
+SUMMARY:New Event
+DESCRIPTION:This has \\" Bad Quotes \\" in it
+ATTENDEE;CN="Wilfredo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:wsanchez
+ @example.com
+ATTENDEE;CN="Cyrus";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;ROLE=REQ-PARTICI
+ PANT:mailto:cdaboo at example.com
+CREATED:20090203T181910Z
+DTEND;TZID=US/Pacific:20090203T130000
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+event01_after = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//Apple Inc.//iCal 3.0//EN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+DTSTART:20071104T020000
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+TZNAME:PST
+TZOFFSETFROM:-0700
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+DTSTART:20070311T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+TZNAME:PDT
+TZOFFSETFROM:-0800
+TZOFFSETTO:-0700
+END:DAYLIGHT
+END:VTIMEZONE
+BEGIN:VEVENT
+UID:1E238CA1-3C95-4468-B8CD-C8A399F78C71
+DTSTART;TZID=US/Pacific:20090203T120000
+DTEND;TZID=US/Pacific:20090203T130000
+ATTENDEE;CN=Wilfredo Sanchez;CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;X-CALENDA
+ RSERVER-EMAIL=wsanchez at example.com:urn:uuid:6423F94A-6B76-4A3A-815B-D52CFD
+ 77935D
+ATTENDEE;CN=Cyrus Daboo;CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;ROLE=REQ-PARTI
+ CIPANT;X-CALENDARSERVER-EMAIL=cdaboo at example.com:urn:uuid:5A985493-EE2C-46
+ 65-94CF-4DFEA3A89500
+CREATED:20090203T181910Z
+DESCRIPTION:This has " Bad Quotes " in it
+DTSTAMP:20090203T181924Z
+ORGANIZER;CN=Cyrus Daboo;X-CALENDARSERVER-EMAIL=cdaboo at example.com:urn:uui
+ d:5A985493-EE2C-4665-94CF-4DFEA3A89500
+SEQUENCE:2
+SUMMARY:New Event
+TRANSP:OPAQUE
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+event02_broken = "Invalid!"
+
+def isValidCTag(value):
+ """
+ Since ctag is generated from datetime.now(), let's make sure that at
+ least the value is zlib compressed XML
+ """
+ try:
+ value = zlib.decompress(value)
+ except zlib.error:
+ return False
+ try:
+ doc = davxml.WebDAVDocument.fromString(value)
+ return True
+ except ValueError:
+ return False
Modified: CalendarServer/trunk/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/util.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/test/util.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -14,7 +14,10 @@
# limitations under the License.
##
+from __future__ import with_statement
+
import os
+import xattr
from twisted.python.failure import Failure
from twisted.internet.defer import succeed, fail
@@ -38,6 +41,96 @@
config.Memcached.ClientEnabled = False
config.Memcached.ServerEnabled = False
+ def createHierarchy(self, structure):
+ root = self.mktemp()
+ os.mkdir(root)
+
+ def createChildren(parent, subStructure):
+ for childName, childStructure in subStructure.iteritems():
+
+ if childName.startswith("@"):
+ continue
+
+ childPath = os.path.join(parent, childName)
+ if childStructure.has_key("@contents"):
+ # This is a file
+ with open(childPath, "w") as child:
+ child.write(childStructure["@contents"])
+
+ else:
+ # This is a directory
+ os.mkdir(childPath)
+ createChildren(childPath, childStructure)
+
+ if childStructure.has_key("@xattrs"):
+ xattrs = childStructure["@xattrs"]
+ for attr, value in xattrs.iteritems():
+ xattr.setxattr(childPath, attr, value)
+
+ createChildren(root, structure)
+ return root
+
+ def verifyHierarchy(self, root, structure):
+
+ def verifyChildren(parent, subStructure):
+
+ actual = set([child for child in os.listdir(parent)])
+
+ for childName, childStructure in subStructure.iteritems():
+
+ if childName.startswith("@"):
+ continue
+
+ if childName in actual:
+ actual.remove(childName)
+
+ childPath = os.path.join(parent, childName)
+
+ if not os.path.exists(childPath):
+ print "Missing:", childPath
+ return False
+
+ if childStructure.has_key("@contents"):
+ # This is a file
+ with open(childPath) as child:
+ contents = child.read()
+ if contents != childStructure["@contents"]:
+ print "Contents mismatch:", childPath
+ print "Expected:\n%s\n\nActual:\n%s\n" % (childStructure["@contents"], contents)
+ return False
+
+ else:
+ # This is a directory
+ if not verifyChildren(childPath, childStructure):
+ return False
+
+ if childStructure.has_key("@xattrs"):
+ xattrs = childStructure["@xattrs"]
+ for attr, value in xattrs.iteritems():
+ if isinstance(value, str):
+ try:
+ if xattr.getxattr(childPath, attr) != value:
+ print "Xattr mismatch:", childPath, attr
+ return False
+ except:
+ return False
+ else: # method
+ if not value(xattr.getxattr(childPath, attr)):
+ return False
+
+ for attr, value in xattr.xattr(childPath).iteritems():
+ if attr not in xattrs:
+ return False
+
+ if actual:
+ # There are unexpected children
+ return False
+
+ return True
+
+ return verifyChildren(root, structure)
+
+
class InMemoryPropertyStore(object):
def __init__(self):
class _FauxPath(object):
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2009-02-27 23:13:19 UTC (rev 3761)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2009-02-27 23:52:42 UTC (rev 3762)
@@ -14,72 +14,421 @@
# limitations under the License.
##
+from __future__ import with_statement
+
from twisted.web2.dav.fileop import rmdir
-from twistedcaldav.config import config
+from twisted.web2.dav import davxml
+from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
from twistedcaldav.log import Logger
-import os
+from twistedcaldav.ical import Component
+from twistedcaldav.scheduling.cuaddress import normalizeCUAddr
+from twistedcaldav import caldavxml
+from calendarserver.tools.util import getDirectory, dummyDirectoryRecord
+import xattr, itertools, os, zlib, hashlib, datetime, pwd, grp
+from zlib import compress, decompress
+from cPickle import loads as unpickle, PicklingError, UnpicklingError
+
log = Logger()
-class UpgradeTheServer(object):
-
- @staticmethod
- def doUpgrade():
-
- UpgradeTheServer._doPrincipalCollectionInMemoryUpgrade()
-
- @staticmethod
- def _doPrincipalCollectionInMemoryUpgrade():
-
- # Look for the /principals/ directory on disk
- old_principals = os.path.join(config.DocumentRoot, "principals")
- if os.path.exists(old_principals):
- # First move the proxy database and rename it
- UpgradeTheServer._doProxyDatabaseMoveUpgrade()
-
- # Now delete the on disk representation of principals
- rmdir(old_principals)
- log.info(
- "Removed the old principal directory at '%s'."
- % (old_principals,)
- )
- @staticmethod
- def _doProxyDatabaseMoveUpgrade():
-
+
+
+#
+# upgrade_to_1
+#
+# Upconverts data from any calendar server version prior to data format 1
+#
+errorOccurred = False
+
+def upgrade_to_1(config):
+
+ global errorOccurred
+ errorOccurred = False
+
+ def fixBadQuotes(data):
+ if (
+ data.find('\\"') != -1 or
+ data.find('\\\r\n "') != -1 or
+ data.find('\r\n \r\n "') != -1
+ ):
+ # Fix by continuously replacing \" with " until no more
+ # replacements occur
+ while True:
+ newData = data.replace('\\"', '"').replace('\\\r\n "', '\r\n "').replace('\r\n \r\n "', '\r\n "')
+ if newData == data:
+ break
+ else:
+ data = newData
+
+ return data, True
+ else:
+ return data, False
+
+
+
+ def normalizeCUAddrs(data, directory):
+ cal = Component.fromString(data)
+
+ def lookupFunction(cuaddr):
+ try:
+ principal = directory.principalForCalendarUserAddress(cuaddr)
+ except Exception, e:
+ log.debug("Lookup of %s failed: %s" % (cuaddr, e))
+ principal = None
+
+ if principal is None:
+ return (None, None, None)
+ else:
+ return (principal.record.fullName.decode("utf-8"),
+ principal.record.guid,
+ principal.record.calendarUserAddresses)
+
+ cal.normalizeCalendarUserAddresses(lookupFunction)
+
+ newData = str(cal)
+ return newData, not newData == data
+
+
+ def upgradeCalendarCollection(calPath, directory):
+
+ global errorOccurred
+
+ collectionUpdated = False
+
+ for resource in os.listdir(calPath):
+
+ if resource.startswith("."):
+ continue
+
+ resPath = os.path.join(calPath, resource)
+
+ if os.path.isdir(resPath):
+ # Skip directories
+ continue
+
+ log.info("Processing: %s" % (resPath,))
+ needsRewrite = False
+ with open(resPath) as res:
+ data = res.read()
+
+ try:
+ data, fixed = fixBadQuotes(data)
+ if fixed:
+ log.info("Fixing bad quotes in %s" % (resPath,))
+ needsRewrite = True
+ except Exception, e:
+ log.error("Error while fixing bad quotes in %s: %s" %
+ (resPath, e))
+ errorOccurred = True
+ continue
+
+ try:
+ data, fixed = normalizeCUAddrs(data, directory)
+ if fixed:
+ log.info("Normalized CUAddrs in %s" % (resPath,))
+ needsRewrite = True
+ except Exception, e:
+ log.error("Error while normalizing %s: %s" %
+ (resPath, e))
+ errorOccurred = True
+ continue
+
+ if needsRewrite:
+ with open(resPath, "w") as res:
+ res.write(data)
+
+ md5value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>%s</getcontentmd5>\r\n" % (hashlib.md5(data).hexdigest(),)
+ md5value = zlib.compress(md5value)
+ xattr.setxattr(resPath, "WebDAV:{http:%2F%2Ftwistedmatrix.com%2Fxml_namespace%2Fdav%2F}getcontentmd5", md5value)
+
+ collectionUpdated = True
+
+
+ if collectionUpdated:
+ ctagValue = "<?xml version='1.0' encoding='UTF-8'?>\r\n<getctag xmlns='http://calendarserver.org/ns/'>%s</getctag>\r\n" % (str(datetime.datetime.now()),)
+ ctagValue = zlib.compress(ctagValue)
+ xattr.setxattr(calPath, "WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag", ctagValue)
+
+
+ def upgradeCalendarHome(homePath, directory):
+
+ log.info("Upgrading calendar home: %s" % (homePath,))
+
+ for cal in os.listdir(homePath):
+ calPath = os.path.join(homePath, cal)
+ log.info("Upgrading calendar: %s" % (calPath,))
+ upgradeCalendarCollection(calPath, directory)
+
+ # Change the calendar-free-busy-set xattrs of the inbox to the
+ # __uids__/<guid> form
+ if cal == "inbox":
+ for attr, value in xattr.xattr(calPath).iteritems():
+ if attr == "WebDAV:{urn:ietf:params:xml:ns:caldav}calendar-free-busy-set":
+ value = updateFreeBusySet(value, directory)
+ if value is not None:
+ # Need to write the xattr back to disk
+ xattr.setxattr(calPath, attr, value)
+
+
+
+
+ def doProxyDatabaseMoveUpgrade(config, uid=-1, gid=-1):
+
# See if the old DB is present
- old_db_path = os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)
- if not os.path.exists(old_db_path):
+ oldDbPath = os.path.join(config.DocumentRoot, "principals",
+ CalendarUserProxyDatabase.dbOldFilename)
+ if not os.path.exists(oldDbPath):
# Nothing to be done
return
-
+
# See if the new one is already present
- new_db_path = os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)
- if os.path.exists(new_db_path):
- # We have a problem - both the old and new ones exist. Stop the server from starting
- # up and alert the admin to this condition
+ newDbPath = os.path.join(config.DataRoot,
+ CalendarUserProxyDatabase.dbFilename)
+ if os.path.exists(newDbPath):
+ # We have a problem - both the old and new ones exist. Stop the server
+ # from starting up and alert the admin to this condition
raise UpgradeError(
"Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' because the new database already exists."
- % (old_db_path, new_db_path,)
+ % (oldDbPath, newDbPath,)
)
-
+
# Now move the old one to the new location
try:
- os.rename(old_db_path, new_db_path)
+ if not os.path.exists(config.DataRoot):
+ makeDirsUserGroup(config.DataRoot, uid=uid, gid=gid)
+ os.rename(oldDbPath, newDbPath)
except Exception, e:
raise UpgradeError(
"Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' due to %s."
- % (old_db_path, new_db_path, str(e))
+ % (oldDbPath, newDbPath, str(e))
)
-
+
log.info(
"Moved the calendar user proxy database from '%s' to '%s'."
- % (old_db_path, new_db_path,)
+ % (oldDbPath, newDbPath,)
)
+
+ def moveCalendarHome(oldHome, newHome, uid=-1, gid=-1):
+ if os.path.exists(newHome):
+ # Both old and new homes exist; stop immediately to let the
+ # administrator fix it
+ raise UpgradeError(
+ "Upgrade Error: calendar home is in two places: %s and %s. Please remove one of them and restart calendar server."
+ % (oldHome, newHome)
+ )
+
+ makeDirsUserGroup(os.path.dirname(newHome.rstrip("/")), uid=uid,
+ gid=gid)
+ os.rename(oldHome, newHome)
+
+
+
+ directory = getDirectory()
+ docRoot = config.DocumentRoot
+
+
+ # Determine uid/gid for ownership of directories we create here
+ uid = -1
+ if config.UserName:
+ try:
+ uid = pwd.getpwnam(config.UserName).pw_uid
+ except KeyError:
+ log.error("User not found: %s" % (config.UserName,))
+
+ gid = -1
+ if config.GroupName:
+ try:
+ gid = grp.getgrnam(config.GroupName).gr_gid
+ except KeyError:
+ log.error("Group not found: %s" % (config.GroupName,))
+
+
+ if os.path.exists(docRoot):
+
+ # Look for the /principals/ directory on disk
+ oldPrincipals = os.path.join(docRoot, "principals")
+ if os.path.exists(oldPrincipals):
+ # First move the proxy database and rename it
+ doProxyDatabaseMoveUpgrade(config, uid=uid, gid=gid)
+
+ # Now delete the on disk representation of principals
+ rmdir(oldPrincipals)
+ log.info(
+ "Removed the old principal directory at '%s'."
+ % (oldPrincipals,)
+ )
+
+ calRoot = os.path.join(docRoot, "calendars")
+ if os.path.exists(calRoot):
+
+ uidHomes = os.path.join(calRoot, "__uids__")
+
+ # Move calendar homes to new location:
+
+ if os.path.exists(uidHomes):
+ for home in os.listdir(uidHomes):
+
+ # MOR: This assumes no UID is going to be 2 chars or less
+ if len(home) <= 2:
+ continue
+
+ oldHome = os.path.join(uidHomes, home)
+ newHome = os.path.join(uidHomes, home[0:2], home[2:4], home)
+ moveCalendarHome(oldHome, newHome, uid=uid, gid=gid)
+
+ else:
+ os.mkdir(uidHomes)
+ os.chown(uidHomes, uid, gid)
+
+ for recordType, dirName in (
+ (DirectoryService.recordType_users, "users"),
+ (DirectoryService.recordType_groups, "groups"),
+ (DirectoryService.recordType_locations, "locations"),
+ (DirectoryService.recordType_resources, "resources"),
+ ):
+ dirPath = os.path.join(calRoot, dirName)
+ if os.path.exists(dirPath):
+ for shortName in os.listdir(dirPath):
+ record = directory.recordWithShortName(recordType,
+ shortName)
+ if record is not None:
+ oldHome = os.path.join(dirPath, shortName)
+ newHome = os.path.join(uidHomes, record.uid[0:2],
+ record.uid[2:4], record.uid)
+ moveCalendarHome(oldHome, newHome, uid=uid, gid=gid)
+ os.rmdir(dirPath)
+
+ # Upgrade calendar homes in the new location:
+ for first in os.listdir(uidHomes):
+ if len(first) == 2:
+ firstPath = os.path.join(uidHomes, first)
+ for second in os.listdir(firstPath):
+ if len(second) == 2:
+ secondPath = os.path.join(firstPath, second)
+ for home in os.listdir(secondPath):
+ homePath = os.path.join(secondPath, home)
+ upgradeCalendarHome(homePath, directory)
+
+ if errorOccurred:
+ raise UpgradeError("Data upgrade failed, see error.log for details")
+
+
+# The on-disk version number (which defaults to zero if .calendarserver_version
+# doesn't exist), is compared with each of the numbers in the upgradeMethods
+# array. If it is less than the number, the associated method is called.
+
+upgradeMethods = [
+ (1, upgrade_to_1),
+]
+
+def upgradeData(config):
+
+ docRoot = config.DocumentRoot
+
+ versionFilePath = os.path.join(docRoot, ".calendarserver_version")
+
+ onDiskVersion = 0
+ if os.path.exists(versionFilePath):
+ try:
+ with open(versionFilePath) as versionFile:
+ onDiskVersion = int(versionFile.read().strip())
+ except IOError, e:
+ log.error("Cannot open %s; skipping migration" %
+ (versionFilePath,))
+ except ValueError, e:
+ log.error("Invalid version number in %s; skipping migration" %
+ (versionFilePath,))
+
+ for version, method in upgradeMethods:
+ if onDiskVersion < version:
+ log.info("Upgrading to version %d" % (version,))
+ method(config)
+ with open(versionFilePath, "w") as verFile:
+ verFile.write(str(version))
+
+
class UpgradeError(RuntimeError):
"""
Generic upgrade error.
"""
pass
+
+
+#
+# Utility functions
+#
+def updateFreeBusyHref(href, directory):
+ pieces = href.split("/")
+ if pieces[2] == "__uids__":
+ # Already updated
+ return None
+
+ recordType = pieces[2]
+ shortName = pieces[3]
+ record = directory.recordWithShortName(recordType, shortName)
+ if record is None:
+ msg = "Can't update free-busy href; %s is not in the directory" % shortName
+ log.error(msg)
+ raise UpgradeError(msg)
+
+ uid = record.uid
+ newHref = "/calendars/__uids__/%s/%s/" % (uid, pieces[4])
+ return newHref
+
+
+def updateFreeBusySet(value, directory):
+
+ try:
+ value = zlib.decompress(value)
+ except zlib.error:
+ # Legacy data - not zlib compressed
+ pass
+
+ try:
+ doc = davxml.WebDAVDocument.fromString(value)
+ freeBusySet = doc.root_element
+ except ValueError:
+ try:
+ freeBusySet = unpickle(value)
+ except UnpicklingError:
+ log.err("Invalid xattr property value for: %s" % attr)
+ # MOR: continue on?
+ return None
+
+ fbset = set()
+ didUpdate = False
+ for href in freeBusySet.children:
+ href = str(href)
+ newHref = updateFreeBusyHref(href, directory)
+ if newHref is None:
+ fbset.add(href)
+ else:
+ didUpdate = True
+ fbset.add(newHref)
+
+ if didUpdate:
+ property = caldavxml.CalendarFreeBusySet(*[davxml.HRef(href)
+ for href in fbset])
+ value = compress(property.toxml())
+ return value
+
+ return None # no update required
+
+
+def makeDirsUserGroup(path, uid=-1, gid=-1):
+ parts = path.split("/")
+ if parts[0] == "": # absolute path
+ parts[0] = "/"
+
+ path = ""
+ for part in parts:
+ if not part:
+ continue
+ path = os.path.join(path, part)
+ if not os.path.exists(path):
+ os.mkdir(path)
+ os.chown(path, uid, gid)
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090227/7feb45b0/attachment-0001.html>
More information about the calendarserver-changes
mailing list