[CalendarServer-changes] [6253] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Sep 7 14:12:43 PDT 2010
Revision: 6253
http://trac.macosforge.org/projects/calendarserver/changeset/6253
Author: sagen at apple.com
Date: 2010-09-07 14:12:42 -0700 (Tue, 07 Sep 2010)
Log Message:
-----------
upgradeData is now deferred, yet still runs prior to child processes being spawned. Locations and resources are migrated from OD at initial startup (iff .calendarserver_version < 2). test_upgrade.py brought back to life.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tools/resources.py
CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
CalendarServer/trunk/twistedcaldav/test/util.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2010-09-07 18:04:33 UTC (rev 6252)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2010-09-07 21:12:42 UTC (rev 6253)
@@ -40,7 +40,7 @@
from twisted.python.usage import Options, UsageError
from twisted.python.reflect import namedClass
from twisted.plugin import IPlugin
-from twisted.internet.defer import Deferred, gatherResults
+from twisted.internet.defer import Deferred, gatherResults, succeed
from twisted.internet import error, reactor
from twisted.internet.reactor import callLater, addSystemEventTrigger
from twisted.internet.process import ProcessExitedAlready
@@ -407,12 +407,6 @@
# Process localization string files
processLocalizationFiles(config.Localization)
- # Now do any on disk upgrades we might need. Note that this
- # will only do the filesystem-format upgrades; migration to the
- # database needs to be done when the connection and possibly
- # server is already up and running. -glyph
- upgradeData(config)
-
# Make sure proxies get initialized
if config.ProxyLoadFromFile:
def _doProxyUpdate():
@@ -744,6 +738,12 @@
"""
s = ErrorLoggingMultiService()
+ # Schedule any on disk upgrades we might need. Note that this
+ # will only do the filesystem-format upgrades; migration to the
+ # database needs to be done when the connection and possibly
+ # server is already up and running. -glyph
+ addSystemEventTrigger("before", "startup", upgradeData, config)
+
# Make sure no old socket files are lying around.
self.deleteStaleSocketFiles()
Modified: CalendarServer/trunk/calendarserver/tools/resources.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/resources.py 2010-09-07 18:04:33 UTC (rev 6252)
+++ CalendarServer/trunk/calendarserver/tools/resources.py 2010-09-07 21:12:42 UTC (rev 6253)
@@ -187,8 +187,8 @@
@inlineCallbacks
-def migrateResources(sourceService, destService, queryMethod=queryForType,
- verbose=False):
+def migrateResources(sourceService, destService, autoSchedules=None,
+ queryMethod=queryForType, verbose=False):
for recordTypeOD, recordType in (
(dsattributes.kDSStdRecordTypeResources, DirectoryService.recordType_resources),
@@ -205,9 +205,13 @@
if record is None:
if verbose:
print "Migrating %s (%s)" % (fullName, recordType)
+ if autoSchedules is not None:
+ autoSchedule = autoSchedules.get(guid, 1)
+ else:
+ autoSchedule = True
yield updateRecord(True, destService, recordType,
guid=guid, shortNames=[recordName], fullName=fullName,
- autoSchedule="true")
+ autoSchedule=autoSchedule)
if __name__ == "__main__":
Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2010-09-07 18:04:33 UTC (rev 6252)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2010-09-07 21:12:42 UTC (rev 6253)
@@ -15,17 +15,15 @@
##
+from twisted.internet.defer import inlineCallbacks, returnValue
from twext.web2.dav import davxml
-
from twistedcaldav.config import config
-# from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
-#from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
-CalendarUserProxyDatabase = None
+from twistedcaldav.directory.calendaruserproxy import ProxySqliteDB
from twistedcaldav.directory.xmlfile import XMLDirectoryService
from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
from twistedcaldav.mail import MailGatewayTokensDatabase
from twistedcaldav.upgrade import UpgradeError, upgradeData, updateFreeBusySet
-#from twistedcaldav.test.util import TestCase
+from twistedcaldav.test.util import TestCase
from calendarserver.tools.util import getDirectory
import hashlib
@@ -35,9 +33,10 @@
cTagAttr = "WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"
md5Attr = "WebDAV:{http:%2F%2Ftwistedmatrix.com%2Fxml_namespace%2Fdav%2F}getcontentmd5"
+OLDPROXYFILE = ".db.calendaruserproxy"
+NEWPROXYFILE = "proxies.sqlite"
-class ProxyDBUpgradeTests(object): #(TestCase):
- todo = "upgrade.py needs to be fixed"
+class UpgradeTests(TestCase):
def setUpXMLDirectory(self):
xmlFile = os.path.join(os.path.dirname(os.path.dirname(__file__)),
@@ -48,7 +47,12 @@
"directory", "test", "augments.xml")
config.AugmentService.params.xmlFiles = (xmlAugmentsFile,)
+ resourceFile = os.path.join(os.path.dirname(os.path.dirname(__file__)),
+ "directory", "test", "resources.xml")
+ config.ResourceService.params.xmlFile = resourceFile
+
+
def setUpInitialStates(self):
self.setUpXMLDirectory()
@@ -62,7 +66,7 @@
def setUpOldDocRoot(self):
# Set up doc root
- self.olddocroot = self.mktemp()
+ self.olddocroot = os.path.abspath(self.mktemp())
os.mkdir(self.olddocroot)
principals = os.path.join(self.olddocroot, "principals")
@@ -74,18 +78,13 @@
os.mkdir(os.path.join(principals, "resources"))
os.mkdir(os.path.join(principals, "sudoers"))
- proxyDB = CalendarUserProxyDatabase(principals)
- proxyDB._db()
- os.rename(
- os.path.join(principals, CalendarUserProxyDatabase.dbFilename),
- os.path.join(principals, CalendarUserProxyDatabase.dbOldFilename),
- )
+ open(os.path.join(principals, OLDPROXYFILE), "w").close()
def setUpOldDocRootWithoutDB(self):
# Set up doc root
- self.olddocrootnodb = self.mktemp()
+ self.olddocrootnodb = os.path.abspath(self.mktemp())
os.mkdir(self.olddocrootnodb)
principals = os.path.join(self.olddocrootnodb, "principals")
@@ -101,7 +100,7 @@
def setUpNewDocRoot(self):
# Set up doc root
- self.newdocroot = self.mktemp()
+ self.newdocroot = os.path.abspath(self.mktemp())
os.mkdir(self.newdocroot)
os.mkdir(os.path.join(self.newdocroot, "calendars"))
@@ -109,18 +108,22 @@
def setUpNewDataRoot(self):
# Set up data root
- self.newdataroot = self.mktemp()
+ self.newdataroot = os.path.abspath(self.mktemp())
os.mkdir(self.newdataroot)
def setUpDataRootWithProxyDB(self):
# Set up data root
- self.existingdataroot = self.mktemp()
+ self.existingdataroot = os.path.abspath(self.mktemp())
os.mkdir(self.existingdataroot)
- proxyDB = CalendarUserProxyDatabase(self.existingdataroot)
- proxyDB._db()
+ principals = os.path.join(self.existingdataroot, "principals")
+ os.mkdir(principals)
+ open(os.path.join(self.existingdataroot, NEWPROXYFILE), "w").close()
+
+
+ @inlineCallbacks
def test_normalUpgrade(self):
"""
Test the behavior of normal upgrade from old server to new.
@@ -135,16 +138,17 @@
# Check pre-conditions
self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
self.assertTrue(os.path.isdir(os.path.join(config.DocumentRoot, "principals")))
- self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
- self.assertFalse(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", OLDPROXYFILE)))
+ self.assertFalse(os.path.exists(os.path.join(config.DataRoot, NEWPROXYFILE)))
- upgradeData(config)
+ (yield upgradeData(config))
# Check post-conditions
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
- self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, NEWPROXYFILE)))
+ @inlineCallbacks
def test_noUpgrade(self):
"""
Test the behavior of running on a new server (i.e. no upgrade needed).
@@ -157,13 +161,13 @@
# Check pre-conditions
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
- self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, NEWPROXYFILE)))
- upgradeData(config)
+ (yield upgradeData(config))
# Check post-conditions
self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
- self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, NEWPROXYFILE)))
def test_freeBusyUpgrade(self):
@@ -198,7 +202,7 @@
# Verify these values do require updating:
#
- expected = "<?xml version='1.0' encoding='UTF-8'?><calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"
+ expected = "<?xml version='1.0' encoding='UTF-8'?>\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"
# Uncompressed XML
value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/wsanchez/calendar</href>\r\n</calendar-free-busy-set>\r\n"
@@ -226,13 +230,14 @@
# Shortname not in directory, return empty string
#
- expected = "<?xml version='1.0' encoding='UTF-8'?><calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'/>"
+ expected = "<?xml version='1.0' encoding='UTF-8'?>\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'/>"
value = "<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/users/nonexistent/calendar</href>\r\n</calendar-free-busy-set>\r\n"
newValue = updateFreeBusySet(value, directory)
newValue = zlib.decompress(newValue)
self.assertEquals(newValue, expected)
+ @inlineCallbacks
def verifyDirectoryComparison(self, before, after, reverify=False):
"""
Verify that the hierarchy described by "before", when upgraded, matches
@@ -254,15 +259,16 @@
config.DocumentRoot = root
config.DataRoot = root
- upgradeData(config)
+ (yield upgradeData(config))
self.assertTrue(self.verifyHierarchy(root, after))
if reverify:
# Ensure that repeating the process doesn't change anything
- upgradeData(config)
+ (yield upgradeData(config))
self.assertTrue(self.verifyHierarchy(root, after))
+ @inlineCallbacks
def test_removeNotificationDirectories(self):
"""
The upgrade process should remove unused notification directories in
@@ -298,17 +304,17 @@
}
},
".calendarserver_version" : {
- "@contents" : "1",
+ "@contents" : "2",
},
- CalendarUserProxyDatabase.dbFilename : { "@contents" : None },
MailGatewayTokensDatabase.dbFilename : { "@contents" : None },
- ResourceInfoDatabase.dbFilename : { "@contents" : None },
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) : { "@contents" : None },
"tasks" : {"incoming" : {}}
}
- self.verifyDirectoryComparison(before, after)
+ (yield self.verifyDirectoryComparison(before, after))
+ @inlineCallbacks
def test_calendarsUpgradeWithTypes(self):
"""
Verify that calendar homes in the /calendars/<type>/<shortname>/ form
@@ -361,7 +367,7 @@
},
"principals" :
{
- CalendarUserProxyDatabase.dbOldFilename :
+ OLDPROXYFILE :
{
"@contents" : "",
}
@@ -377,7 +383,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -408,7 +414,7 @@
{
"@xattrs" :
{
- freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?><calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"),
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"),
},
},
},
@@ -428,7 +434,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -436,15 +442,16 @@
{
"@contents" : None,
},
-# ResourceInfoDatabase.dbFilename :
-# {
-# "@contents" : None,
-# }
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after, reverify=True)
+ (yield self.verifyDirectoryComparison(before, after, reverify=True))
+ @inlineCallbacks
def test_calendarsUpgradeWithOrphans(self):
"""
Verify that calendar homes in the /calendars/<type>/<shortname>/ form
@@ -471,7 +478,7 @@
},
"principals" :
{
- CalendarUserProxyDatabase.dbOldFilename :
+ OLDPROXYFILE :
{
"@contents" : "",
}
@@ -496,7 +503,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -504,7 +511,7 @@
{
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -512,15 +519,16 @@
{
"@contents" : None,
},
- ResourceInfoDatabase.dbFilename :
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
{
- "@contents" : None,
- }
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after, reverify=True)
+ (yield self.verifyDirectoryComparison(before, after, reverify=True))
+ @inlineCallbacks
def test_calendarsUpgradeWithDuplicateOrphans(self):
"""
Verify that calendar homes in the /calendars/<type>/<shortname>/ form
@@ -556,7 +564,7 @@
},
"principals" :
{
- CalendarUserProxyDatabase.dbOldFilename :
+ OLDPROXYFILE :
{
"@contents" : "",
}
@@ -587,7 +595,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -595,7 +603,7 @@
{
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -603,15 +611,16 @@
{
"@contents" : None,
},
- ResourceInfoDatabase.dbFilename :
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
{
- "@contents" : None,
- }
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after, reverify=True)
+ (yield self.verifyDirectoryComparison(before, after, reverify=True))
+ @inlineCallbacks
def test_calendarsUpgradeWithUnknownFiles(self):
"""
Unknown files, including .DS_Store files at any point in the hierarchy,
@@ -660,7 +669,7 @@
{
"@contents" : "",
},
- CalendarUserProxyDatabase.dbOldFilename :
+ OLDPROXYFILE :
{
"@contents" : "",
}
@@ -680,7 +689,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -690,7 +699,7 @@
},
"__uids__" : ignoredUIDContents,
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -698,15 +707,16 @@
{
"@contents" : None,
},
- ResourceInfoDatabase.dbFilename :
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
{
- "@contents" : None,
- }
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after, reverify=True)
+ (yield self.verifyDirectoryComparison(before, after, reverify=True))
+ @inlineCallbacks
def test_calendarsUpgradeWithUIDs(self):
"""
Verify that calendar homes in the /calendars/__uids__/<guid>/ form
@@ -742,7 +752,7 @@
},
"principals" :
{
- CalendarUserProxyDatabase.dbOldFilename :
+ OLDPROXYFILE :
{
"@contents" : "",
}
@@ -758,7 +768,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -785,7 +795,7 @@
{
"@xattrs" :
{
- freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?><calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"),
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"),
},
},
},
@@ -793,7 +803,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -801,15 +811,16 @@
{
"@contents" : None,
},
-# ResourceInfoDatabase.dbFilename :
-# {
-# "@contents" : None,
-# }
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after, reverify=True)
+ (yield self.verifyDirectoryComparison(before, after, reverify=True))
+ @inlineCallbacks
def test_calendarsUpgradeWithUIDsMultilevel(self):
"""
Verify that calendar homes in the /calendars/__uids__/XX/YY/<guid>/
@@ -858,7 +869,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : "",
}
@@ -873,7 +884,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -905,7 +916,7 @@
{
"@xattrs" :
{
- freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?><calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"),
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>"),
},
},
},
@@ -913,7 +924,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -921,14 +932,15 @@
{
"@contents" : None,
},
-# ResourceInfoDatabase.dbFilename :
-# {
-# "@contents" : None,
-# }
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after, reverify=True)
+ (yield self.verifyDirectoryComparison(before, after, reverify=True))
+ @inlineCallbacks
def test_calendarsUpgradeWithNoChange(self):
"""
Verify that calendar homes in the /calendars/__uids__/XX/YY/<guid>/
@@ -977,7 +989,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : "",
}
@@ -992,7 +1004,7 @@
},
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
"calendars" :
{
@@ -1032,7 +1044,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -1040,15 +1052,16 @@
{
"@contents" : None,
},
-# ResourceInfoDatabase.dbFilename :
-# {
-# "@contents" : None,
-# }
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None
+ },
}
- self.verifyDirectoryComparison(before, after)
+ (yield self.verifyDirectoryComparison(before, after))
+ @inlineCallbacks
def test_calendarsUpgradeWithError(self):
"""
Verify that a problem with one resource doesn't stop the process, but
@@ -1084,7 +1097,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : "",
}
@@ -1124,7 +1137,7 @@
},
},
},
- CalendarUserProxyDatabase.dbFilename :
+ NEWPROXYFILE :
{
"@contents" : None,
},
@@ -1132,10 +1145,10 @@
{
"@contents" : None,
},
-# ResourceInfoDatabase.dbFilename :
-# {
-# "@contents" : None,
-# }
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None
+ },
}
@@ -1144,9 +1157,16 @@
config.DocumentRoot = root
config.DataRoot = root
- self.assertRaises(UpgradeError, upgradeData, config)
+ try:
+ (yield upgradeData(config))
+ except UpgradeError:
+ pass
+ else:
+ self.fail("UpgradeError expected")
+
self.assertTrue(self.verifyHierarchy(root, after))
+ @inlineCallbacks
def test_migrateResourceInfo(self):
# Fake getResourceInfo( )
@@ -1165,59 +1185,69 @@
self.setUpInitialStates()
# Override the normal getResourceInfo method with our own:
- XMLDirectoryService.getResourceInfo = _getResourceInfo
+ # XMLDirectoryService.getResourceInfo = _getResourceInfo
+ self.patch(XMLDirectoryService, "getResourceInfo", _getResourceInfo)
before = { }
after = {
".calendarserver_version" :
{
- "@contents" : "1",
+ "@contents" : "2",
},
-# CalendarUserProxyDatabase.dbFilename :
-# {
-# "@contents" : None,
-# },
+ NEWPROXYFILE :
+ {
+ "@contents" : None,
+ },
MailGatewayTokensDatabase.dbFilename :
{
"@contents" : None,
},
-# ResourceInfoDatabase.dbFilename :
-# {
-# "@contents" : None,
-# }
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None
+ },
+ ResourceInfoDatabase.dbFilename :
+ {
+ "@contents" : None,
+ },
+ "%s-journal" % (ResourceInfoDatabase.dbFilename,) :
+ {
+ "@contents" : None,
+ }
}
root = self.createHierarchy(before)
config.DocumentRoot = root
config.DataRoot = root
- upgradeData(config)
+ (yield upgradeData(config))
self.assertTrue(self.verifyHierarchy(root, after))
-# calendarUserProxyDatabase = CalendarUserProxyDatabase(root)
-# resourceInfoDatabase = ResourceInfoDatabase(root)
-#
-# for guid, info in assignments.iteritems():
-#
-# proxyGroup = "%s#calendar-proxy-write" % (guid,)
-# result = set([row[0] for row in calendarUserProxyDatabase._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", proxyGroup)])
-# if info[1]:
-# self.assertTrue(info[1] in result)
-# else:
-# self.assertTrue(not result)
-#
-# readOnlyProxyGroup = "%s#calendar-proxy-read" % (guid,)
-# result = set([row[0] for row in calendarUserProxyDatabase._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", readOnlyProxyGroup)])
-# if info[2]:
-# self.assertTrue(info[2] in result)
-# else:
-# self.assertTrue(not result)
-#
-# autoSchedule = resourceInfoDatabase._db_value_for_sql("select AUTOSCHEDULE from RESOURCEINFO where GUID = :1", guid)
-# autoSchedule = autoSchedule == 1
-# self.assertEquals(info[0], autoSchedule)
+ calendarUserProxyDatabase = ProxySqliteDB(NEWPROXYFILE)
+ resourceInfoDatabase = ResourceInfoDatabase(root)
+ for guid, info in assignments.iteritems():
+ proxyGroup = "%s#calendar-proxy-write" % (guid,)
+ result = set([row[0] for row in calendarUserProxyDatabase._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", proxyGroup)])
+ if info[1]:
+ self.assertTrue(info[1] in result)
+ else:
+ self.assertTrue(not result)
+ readOnlyProxyGroup = "%s#calendar-proxy-read" % (guid,)
+ result = set([row[0] for row in calendarUserProxyDatabase._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", readOnlyProxyGroup)])
+ if info[2]:
+ self.assertTrue(info[2] in result)
+ else:
+ self.assertTrue(not result)
+
+ autoSchedule = resourceInfoDatabase._db_value_for_sql("select AUTOSCHEDULE from RESOURCEINFO where GUID = :1", guid)
+ autoSchedule = autoSchedule == 1
+ self.assertEquals(info[0], autoSchedule)
+
+ test_migrateResourceInfo.todo = "FIXME: perhaps ProxySqliteDB isn't being set up correctly?"
+
+
event01_before = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 3.0//EN
Modified: CalendarServer/trunk/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/util.py 2010-09-07 18:04:33 UTC (rev 6252)
+++ CalendarServer/trunk/twistedcaldav/test/util.py 2010-09-07 21:12:42 UTC (rev 6253)
@@ -165,7 +165,7 @@
def createHierarchy(self, structure, root=None):
if root is None:
- root = self.mktemp()
+ root = os.path.abspath(self.mktemp())
os.mkdir(root)
def createChildren(parent, subStructure):
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2010-09-07 18:04:33 UTC (rev 6252)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2010-09-07 21:12:42 UTC (rev 6253)
@@ -26,13 +26,19 @@
from twext.python.log import Logger
+from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
+from twistedcaldav.directory.xmlfile import XMLDirectoryService
+from twistedcaldav.directory.calendaruserproxy import ProxySqliteDB
from twistedcaldav.directory.directory import DirectoryService
-#from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
+from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
from twistedcaldav.mail import MailGatewayTokensDatabase
from twistedcaldav.ical import Component
from twistedcaldav import caldavxml
+from twisted.internet.defer import inlineCallbacks, succeed, returnValue
+
from calendarserver.tools.util import getDirectory
+from calendarserver.tools.resources import migrateResources
log = Logger()
@@ -210,44 +216,44 @@
def doProxyDatabaseMoveUpgrade(config, uid=-1, gid=-1):
- pass
-# # See if the new one is already present
-# newDbPath = os.path.join(config.DataRoot,
-# CalendarUserProxyDatabase.dbFilename)
-# if os.path.exists(newDbPath):
-# # Nothing to be done, it's already in the new location
-# return
-#
-# # See if the old DB is present
-# oldDbPath = os.path.join(config.DocumentRoot, "principals",
-# CalendarUserProxyDatabase.dbOldFilename)
-# if not os.path.exists(oldDbPath):
-# # Nothing to be moved
-# return
-#
-# # Now move the old one to the new location
-# try:
-# if not os.path.exists(config.DataRoot):
-# makeDirsUserGroup(config.DataRoot, uid=uid, gid=gid)
-# try:
-# os.rename(oldDbPath, newDbPath)
-# except OSError:
-# # Can't rename, must copy/delete
-# shutil.copy2(oldDbPath, newDbPath)
-# os.remove(oldDbPath)
-#
-# except Exception, e:
-# raise UpgradeError(
-# "Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' due to %s."
-# % (oldDbPath, newDbPath, str(e))
-# )
-#
-# log.debug(
-# "Moved the calendar user proxy database from '%s' to '%s'."
-# % (oldDbPath, newDbPath,)
-# )
+ # See if the new one is already present
+ oldFilename = ".db.calendaruserproxy"
+ newFilename = "proxies.sqlite"
+ newDbPath = os.path.join(config.DataRoot, newFilename)
+ if os.path.exists(newDbPath):
+ # Nothing to be done, it's already in the new location
+ return
+ # See if the old DB is present
+ oldDbPath = os.path.join(config.DocumentRoot, "principals", oldFilename)
+ if not os.path.exists(oldDbPath):
+ # Nothing to be moved
+ return
+
+ # Now move the old one to the new location
+ try:
+ if not os.path.exists(config.DataRoot):
+ makeDirsUserGroup(config.DataRoot, uid=uid, gid=gid)
+ try:
+ os.rename(oldDbPath, newDbPath)
+ except OSError:
+ # Can't rename, must copy/delete
+ shutil.copy2(oldDbPath, newDbPath)
+ os.remove(oldDbPath)
+
+ except Exception, e:
+ raise UpgradeError(
+ "Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' due to %s."
+ % (oldDbPath, newDbPath, str(e))
+ )
+
+ log.debug(
+ "Moved the calendar user proxy database from '%s' to '%s'."
+ % (oldDbPath, newDbPath,)
+ )
+
+
def moveCalendarHome(oldHome, newHome, uid=-1, gid=-1):
if os.path.exists(newHome):
# Both old and new homes exist; stop immediately to let the
@@ -263,35 +269,32 @@
def migrateResourceInfo(config, directory, uid, gid):
- # TODO: we need to account for the new augments database. This means migrating from the pre-resource info
- # implementation and the resource-info implementation
- pass
+ log.info("Fetching delegate assignments and auto-schedule settings from directory")
+ resourceInfoDatabase = ResourceInfoDatabase(config.DataRoot)
+ calendarUserProxyDatabase = ProxySqliteDB(**config.ProxyDBService.params)
+ resourceInfo = directory.getResourceInfo()
+ for guid, autoSchedule, proxy, readOnlyProxy in resourceInfo:
+ resourceInfoDatabase.setAutoScheduleInDatabase(guid, autoSchedule)
+ if proxy:
+ calendarUserProxyDatabase.setGroupMembersInDatabase(
+ "%s#calendar-proxy-write" % (guid,),
+ [proxy]
+ )
+ if readOnlyProxy:
+ calendarUserProxyDatabase.setGroupMembersInDatabase(
+ "%s#calendar-proxy-read" % (guid,),
+ [readOnlyProxy]
+ )
-# log.info("Fetching delegate assignments and auto-schedule settings from directory")
-# resourceInfoDatabase = ResourceInfoDatabase(config.DataRoot)
-# calendarUserProxyDatabase = CalendarUserProxyDatabase(config.DataRoot)
-# resourceInfo = directory.getResourceInfo()
-# for guid, autoSchedule, proxy, readOnlyProxy in resourceInfo:
-# resourceInfoDatabase.setAutoScheduleInDatabase(guid, autoSchedule)
-# if proxy:
-# calendarUserProxyDatabase.setGroupMembersInDatabase(
-# "%s#calendar-proxy-write" % (guid,),
-# [proxy]
-# )
-# if readOnlyProxy:
-# calendarUserProxyDatabase.setGroupMembersInDatabase(
-# "%s#calendar-proxy-read" % (guid,),
-# [readOnlyProxy]
-# )
-#
-# dbPath = os.path.join(config.DataRoot, ResourceInfoDatabase.dbFilename)
-# if os.path.exists(dbPath):
-# os.chown(dbPath, uid, gid)
-#
-# dbPath = os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)
-# if os.path.exists(dbPath):
-# os.chown(dbPath, uid, gid)
+ dbPath = os.path.join(config.DataRoot, ResourceInfoDatabase.dbFilename)
+ if os.path.exists(dbPath):
+ os.chown(dbPath, uid, gid)
+ dbPath = os.path.join(config.DataRoot, "proxies.sqlite")
+ if os.path.exists(dbPath):
+ os.chown(dbPath, uid, gid)
+
+
def createMailTokensDatabase(config, uid, gid):
# Cause the tokens db to be created on disk so we can set the
# permissions on it now
@@ -457,15 +460,55 @@
if errorOccurred:
raise UpgradeError("Data upgrade failed, see error.log for details")
+ return succeed(None)
+
+def upgrade_to_2(config):
+ #
+ # Migrates locations and resources from OD
+ #
+
+ directory = getDirectory()
+ userService = directory.serviceForRecordType("users")
+ resourceService = directory.serviceForRecordType("resources")
+ if (
+ not isinstance(userService, OpenDirectoryService) or
+ not isinstance(resourceService, XMLDirectoryService)
+ ):
+ # Configuration requires no migration
+ return succeed(None)
+
+ # Fetch the autoSchedule assignments from resourceinfo.sqlite and pass
+ # those to migrateResources
+ autoSchedules = {}
+ dbPath = os.path.join(config.DataRoot, ResourceInfoDatabase.dbFilename)
+ if os.path.exists(dbPath):
+ resourceInfoDatabase = ResourceInfoDatabase(config.DataRoot)
+ results = resourceInfoDatabase._db_execute(
+ "select GUID, AUTOSCHEDULE from RESOURCEINFO"
+ )
+ for guid, autoSchedule in results:
+ autoSchedules[guid] = autoSchedule
+
+ # Create internal copies of resources and locations based on what is found
+ # in OD, overriding the autoSchedule default with existing assignments
+ # from resourceinfo.sqlite
+ return migrateResources(userService, resourceService,
+ autoSchedules=autoSchedules)
+
+
+
+
# The on-disk version number (which defaults to zero if .calendarserver_version
# doesn't exist), is compared with each of the numbers in the upgradeMethods
# array. If it is less than the number, the associated method is called.
upgradeMethods = [
(1, upgrade_to_1),
+ (2, upgrade_to_2),
]
+ at inlineCallbacks
def upgradeData(config):
docRoot = config.DocumentRoot
@@ -489,7 +532,7 @@
for version, method in upgradeMethods:
if onDiskVersion < version:
log.warn("Upgrading to version %d" % (version,))
- method(config)
+ (yield method(config))
with open(versionFilePath, "w") as verFile:
verFile.write(str(version))
os.chown(versionFilePath, uid, gid)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100907/1ebaa19e/attachment-0001.html>
More information about the calendarserver-changes
mailing list