[CalendarServer-changes] [2613] CalendarServer/branches/users/cdaboo/upgrade-2610
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jun 20 10:55:47 PDT 2008
Revision: 2613
http://trac.macosforge.org/projects/calendarserver/changeset/2613
Author: cdaboo at apple.com
Date: 2008-06-20 10:55:46 -0700 (Fri, 20 Jun 2008)
Log Message:
-----------
Basically working, but something needs to be done to figure out how to better prevent race conditions
with clusters. Plus the upgrade only option needs a more clean exit rather than an exception.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/upgrade-2610/bin/caldavd
CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/tap.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/test/test_upgrade.py
CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/upgrade.py
Modified: CalendarServer/branches/users/cdaboo/upgrade-2610/bin/caldavd
===================================================================
--- CalendarServer/branches/users/cdaboo/upgrade-2610/bin/caldavd 2008-06-20 17:54:14 UTC (rev 2612)
+++ CalendarServer/branches/users/cdaboo/upgrade-2610/bin/caldavd 2008-06-20 17:55:46 UTC (rev 2613)
@@ -80,15 +80,17 @@
if [ "${1--}" != "-" ]; then echo "$@"; echo; fi;
- echo "Usage: ${program} [-hX] [-u username] [-g groupname] [-T twistd] [-t type] [-f caldavd.plist] [-p statsfile]";
+ echo "Usage: ${program} [-hXU] [-u username] [-g groupname] [-T twistd] [-t type]";
+ echo " [-f caldavd.plist] [-p statsfile] [-R reactor]";
echo "Options:";
echo " -h Print this help and exit";
echo " -X Do not daemonize";
+ echo " -U Upgrade Only";
echo " -u User name to run as";
echo " -g Group name to run as";
+ echo " -T Path to twistd binary";
+ echo " -t Process type (Master, Slave, Single or Combined)";
echo " -f Configuration file to read";
- echo " -T Path to twistd binary";
- echo " -t Process type (Master, Slave or Combined)";
echo " -p Path to the desired pstats file.";
echo " -R The Twisted Reactor to run [${reactor}]";
@@ -101,6 +103,7 @@
'?') usage; ;;
'h') usage -; exit 0; ;;
'X') daemonize="-n"; ;;
+ 'U') upgradeonly="-U"; ;;
'f') configfile="-f ${OPTARG}"; ;;
'T') twistdpath="${OPTARG}"; ;;
'u') username="-u ${OPTARG}"; ;;
@@ -118,6 +121,6 @@
export PYTHONPATH
-echo exec "${python}" "${twistdpath}" "${twistd_reactor}" ${daemonize} ${username} ${groupname} "${plugin_name}" ${configfile} ${service_type} ${profile} "${child_reactor}";
+echo exec "${python}" "${twistdpath}" "${twistd_reactor}" ${daemonize} ${upgradeonly} ${username} ${groupname} "${plugin_name}" ${configfile} ${service_type} ${profile} "${child_reactor}";
-exec "${python}" "${twistdpath}" ${twistd_reactor} ${daemonize} ${username} ${groupname} "${plugin_name}" ${configfile} ${service_type} ${profile} ${child_reactor};
+exec "${python}" "${twistdpath}" ${twistd_reactor} ${daemonize} ${upgradeonly} ${username} ${groupname} "${plugin_name}" ${configfile} ${service_type} ${profile} ${child_reactor};
Modified: CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/directory/calendaruserproxy.py 2008-06-20 17:54:14 UTC (rev 2612)
+++ CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/directory/calendaruserproxy.py 2008-06-20 17:55:46 UTC (rev 2613)
@@ -336,6 +336,7 @@
dbType = "CALENDARUSERPROXY"
dbFilename = "calendaruserproxy.sqlite"
+ dbOldFilename = db_prefix + "calendaruserproxy"
dbFormatVersion = "4"
class ProxyDBMemcacher(Memcacher):
Modified: CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/tap.py
===================================================================
--- CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/tap.py 2008-06-20 17:54:14 UTC (rev 2612)
+++ CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/tap.py 2008-06-20 17:55:46 UTC (rev 2613)
@@ -54,6 +54,7 @@
from twistedcaldav.static import CalendarHomeProvisioningFile
from twistedcaldav.static import TimezoneServiceFile
from twistedcaldav.timezones import TimezoneCache
+from twistedcaldav.upgrade import UpgradeTheServer
from twistedcaldav import pdmonster
from twistedcaldav import memcachepool
@@ -80,9 +81,12 @@
class CalDAVOptions(Options):
- optParameters = [[
- "config", "f", "/etc/caldavd/caldavd.plist", "Path to configuration file."
- ]]
+ optFlags = [
+ ["upgradeonly", "U", "Do server upgrade only."],
+ ]
+ optParameters = [
+ ["config", "f", "/etc/caldavd/caldavd.plist", "Path to configuration file."],
+ ]
zsh_actions = {"config" : "_files -g '*.plist'"}
@@ -708,6 +712,13 @@
makeService_Single = makeService_Slave
def makeService(self, options):
+
+ # Check for upgrade only
+ if options.has_key('upgradeonly'):
+ # Now do any on disk upgrades we might need.
+ UpgradeTheServer.doUpgrade()
+ return None
+
serverType = config.ProcessType
serviceMethod = getattr(self, "makeService_%s" % (serverType,), None)
@@ -715,10 +726,11 @@
if not serviceMethod:
raise UsageError(
"Unknown server type %s. "
- "Please choose: Master, Slave or Combined"
+ "Please choose: Master, Slave, Single or Combined"
% (serverType,)
)
else:
+
service = serviceMethod(options)
#
Added: CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/test/test_upgrade.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/test/test_upgrade.py 2008-06-20 17:55:46 UTC (rev 2613)
@@ -0,0 +1,185 @@
+##
+# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+from twistedcaldav.config import config
+from twistedcaldav.upgrade import UpgradeTheServer
+from twistedcaldav.upgrade import UpgradeError
+
+import os
+
+from twisted.trial.unittest import TestCase
+
+class ProxyDBUpgradeTests(TestCase):
+
+ def setUpInitialStates(self):
+
+ self.setUpOldDocRoot()
+ self.setUpOldDocRootWithoutDB()
+ self.setUpNewDocRoot()
+
+ self.setUpNewDataRoot()
+ self.setUpDataRootWithProxyDB()
+
+ def setUpOldDocRoot(self):
+
+ # Set up doc root
+ self.olddocroot = self.mktemp()
+ os.mkdir(self.olddocroot)
+
+ principals = os.path.join(self.olddocroot, "principals")
+ os.mkdir(principals)
+ os.mkdir(os.path.join(principals, "__uids__"))
+ os.mkdir(os.path.join(principals, "users"))
+ os.mkdir(os.path.join(principals, "groups"))
+ os.mkdir(os.path.join(principals, "locations"))
+ os.mkdir(os.path.join(principals, "resources"))
+ os.mkdir(os.path.join(principals, "sudoers"))
+ os.mkdir(os.path.join(self.olddocroot, "calendars"))
+
+ proxyDB = CalendarUserProxyDatabase(principals)
+ proxyDB._db()
+ os.rename(
+ os.path.join(principals, CalendarUserProxyDatabase.dbFilename),
+ os.path.join(principals, CalendarUserProxyDatabase.dbOldFilename),
+ )
+
+ def setUpOldDocRootWithoutDB(self):
+
+ # Set up doc root
+ self.olddocrootnodb = self.mktemp()
+ os.mkdir(self.olddocrootnodb)
+
+ principals = os.path.join(self.olddocrootnodb, "principals")
+ os.mkdir(principals)
+ os.mkdir(os.path.join(principals, "__uids__"))
+ os.mkdir(os.path.join(principals, "users"))
+ os.mkdir(os.path.join(principals, "groups"))
+ os.mkdir(os.path.join(principals, "locations"))
+ os.mkdir(os.path.join(principals, "resources"))
+ os.mkdir(os.path.join(principals, "sudoers"))
+ os.mkdir(os.path.join(self.olddocrootnodb, "calendars"))
+
+ def setUpNewDocRoot(self):
+
+ # Set up doc root
+ self.newdocroot = self.mktemp()
+ os.mkdir(self.newdocroot)
+
+ os.mkdir(os.path.join(self.newdocroot, "calendars"))
+
+ def setUpNewDataRoot(self):
+
+ # Set up data root
+ self.newdataroot = self.mktemp()
+ os.mkdir(self.newdataroot)
+
+ def setUpDataRootWithProxyDB(self):
+
+ # Set up data root
+ self.existingdataroot = self.mktemp()
+ os.mkdir(self.existingdataroot)
+
+ proxyDB = CalendarUserProxyDatabase(self.existingdataroot)
+ proxyDB._db()
+
+ def test_normalUpgrade(self):
+ """
+ Test the behavior of normal upgrade from old server to new.
+ """
+
+ self.setUpInitialStates()
+
+ config.DocumentRoot = self.olddocroot
+ config.DataRoot = self.newdataroot
+
+ # Check pre-conditions
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.isdir(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
+ self.assertFalse(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ UpgradeTheServer.doUpgrade()
+
+ # Check post-conditions
+ self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ def test_partialUpgrade(self):
+ """
+ Test the behavior of a partial upgrade (one where /principals exists but the proxy db does not) from old server to new.
+ """
+
+ self.setUpInitialStates()
+
+ config.DocumentRoot = self.olddocrootnodb
+ config.DataRoot = self.newdataroot
+
+ # Check pre-conditions
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.isdir(os.path.join(config.DocumentRoot, "principals")))
+ self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
+ self.assertFalse(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ UpgradeTheServer.doUpgrade()
+
+ # Check post-conditions
+ self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
+ self.assertFalse(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ def test_noUpgrade(self):
+ """
+ Test the behavior of running on a new server (i.e. no upgrade needed).
+ """
+
+ self.setUpInitialStates()
+
+ config.DocumentRoot = self.newdocroot
+ config.DataRoot = self.existingdataroot
+
+ # Check pre-conditions
+ self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ UpgradeTheServer.doUpgrade()
+
+ # Check post-conditions
+ self.assertFalse(os.path.exists(os.path.join(config.DocumentRoot, "principals",)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ def test_failedUpgrade(self):
+ """
+ Test the behavior of failed upgrade from old server to new where proxy DB exists in two locations.
+ """
+
+ self.setUpInitialStates()
+
+ config.DocumentRoot = self.olddocroot
+ config.DataRoot = self.existingdataroot
+
+ # Check pre-conditions
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.isdir(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
+ self.assertRaises(UpgradeError, UpgradeTheServer.doUpgrade)
+
+ # Check post-conditions
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.isdir(os.path.join(config.DocumentRoot, "principals")))
+ self.assertTrue(os.path.exists(os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)))
+ self.assertTrue(os.path.exists(os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)))
+
\ No newline at end of file
Added: CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/upgrade.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/upgrade-2610/twistedcaldav/upgrade.py 2008-06-20 17:55:46 UTC (rev 2613)
@@ -0,0 +1,84 @@
+##
+# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.web2.dav.fileop import rmdir
+from twistedcaldav.config import config
+from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+from twistedcaldav.log import Logger
+import os
+
+log = Logger()
+
+class UpgradeTheServer(object):
+
+ @staticmethod
+ def doUpgrade():
+
+ UpgradeTheServer._doPrincipalCollectionInMemoryUpgrade()
+
+ @staticmethod
+ def _doPrincipalCollectionInMemoryUpgrade():
+
+ # Look for the /principals/ directory on disk
+ old_principals = os.path.join(config.DocumentRoot, "principals")
+ if os.path.exists(old_principals):
+ # First move the proxy database and rename it
+ UpgradeTheServer._doProxyDatabaseMoveUpgrade()
+
+ # Now delete the on disk representation of principals
+ rmdir(old_principals)
+ log.info(
+ "Removed the old principal directory at '%s'."
+ % (old_principals,)
+ )
+
+ @staticmethod
+ def _doProxyDatabaseMoveUpgrade():
+
+ # See if the old DB is present
+ old_db_path = os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)
+ if not os.path.exists(old_db_path):
+ # Nothing to be done
+ return
+
+ # See if the new one is already present
+ new_db_path = os.path.join(config.DataRoot, CalendarUserProxyDatabase.dbFilename)
+ if os.path.exists(new_db_path):
+ # We have a problem - both the old and new ones exist. Stop the server from starting
+ # up and alert the admin to this condition
+ raise UpgradeError(
+ "Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' because the new database already exists."
+ % (old_db_path, new_db_path,)
+ )
+
+ # Now move the old one to the new location
+ try:
+ os.rename(old_db_path, new_db_path)
+ except Exception, e:
+ raise UpgradeError(
+ "Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' due to %s."
+ % (old_db_path, new_db_path, str(e))
+ )
+
+ log.info(
+ "Moved the calendar user proxy database from '%s' to '%s'."
+ % (old_db_path, new_db_path,)
+ )
+
+class UpgradeError(RuntimeError):
+ """
+ Generic upgrade error.
+ """
\ No newline at end of file
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080620/aa357bbe/attachment-0001.htm
More information about the calendarserver-changes
mailing list