[CalendarServer-changes] [8258] CalendarServer/branches/users/cdaboo/component-set-fixes

source_changes at macosforge.org source_changes at macosforge.org
Fri Nov 4 20:51:04 PDT 2011


Revision: 8258
          http://trac.macosforge.org/projects/calendarserver/changeset/8258
Author:   cdaboo at apple.com
Date:     2011-11-04 20:51:02 -0700 (Fri, 04 Nov 2011)
Log Message:
-----------
Checkpoint: implements a separate data upgrade service that can incrementally upgrade each calendar home. This is
used with the changes to support supported-component-set and split calendars. More unit tests still need, but basic
functionality is in place.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py

Added Paths:
-----------
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/__init__.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_1_to_2.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/util.py

Removed Paths:
-------------
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py
    CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -58,7 +58,8 @@
 from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
 
 from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService
-from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService
+from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService,\
+    UpgradeDatabaseDataService
 
 from twistedcaldav.config import ConfigurationError
 from twistedcaldav.config import config
@@ -970,9 +971,12 @@
             mainService = createMainService(cp, store)
             upgradeSvc = UpgradeFileSystemFormatService(config,
                 UpgradeDatabaseSchemaService.wrapService(
-                    UpgradeToDatabaseService.wrapService(
-                        CachingFilePath(config.DocumentRoot),
-                        PostDBImportService(config, store, mainService),
+                    UpgradeDatabaseDataService.wrapService(
+                        UpgradeToDatabaseService.wrapService(
+                            CachingFilePath(config.DocumentRoot),
+                            PostDBImportService(config, store, mainService),
+                            store, uid=uid, gid=gid
+                        ),
                         store, uid=uid, gid=gid
                     ),
                     store, uid=uid, gid=gid

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -229,8 +229,23 @@
             
         yield self.createCalendarWithName("inbox")
 
+    @inlineCallbacks
+    def splitCalendars(self):
+        """
+        Split all regular calendars by component type
+        """
+        
+        # Make sure the loop does not operate on any new calendars created during the loop
+        self.log_warn("Splitting calendars for user %s" % (self._ownerUID,))
+        calendars = yield self.calendars()
+        for calendar in calendars:
+            
+            # Ignore inbox - also shared calendars are not part of .calendars() 
+            if calendar.name() == "inbox":
+                continue
+            split_count = yield calendar.splitCollectionByComponentTypes()
+            self.log_warn("  Calendar: '%s', split into %d" % (calendar.name(), split_count+1,))
 
-
 class Calendar(CommonHomeChild):
     """
     File-based implementation of L{ICalendar}.
@@ -386,19 +401,20 @@
         """
         If the calendar contains iCalendar data with different component types, then split it into separate collections
         each containing only one component type. When doing this make sure properties and sharing state are preserved
-        on any new calendars created. Also restrict the new calendars to only the one appropriate component type.
+        on any new calendars created. Also restrict the new calendars to only the one appropriate component type. Return
+        the number of splits done.
         """
         
         # First see how many different component types there are
+        split_count = 0
         components = yield self._countComponentTypes()
         if len(components) <= 1:
 
             # Restrict calendar to single component type
-            if components:
-                component = components[0][0] if components else "VEVENT"
+            component = components[0][0] if components else "VEVENT"
             yield self.setSupportedComponents(component.upper())
 
-            returnValue(None)
+            returnValue(split_count)
         
         # We will leave the component type with the highest count in the current calendar and create new calendars
         # for the others which will be moved over
@@ -407,11 +423,14 @@
         for component, _ignore_count in components:
             if component == maxComponent:
                 continue
+            split_count += 1
             yield self._splitComponentType(component)
 
         # Restrict calendar to single component type
         yield self.setSupportedComponents(maxComponent.upper())
 
+        returnValue(split_count)
+
     @inlineCallbacks
     def _countComponentTypes(self):
         """

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -262,23 +262,20 @@
 
 
     @classproperty
-    def _schemaVersion(cls): #@NoSelf
+    def _calendarserver(cls): #@NoSelf
         cs = schema.CALENDARSERVER
         return Select(
             [cs.VALUE,],
             From=cs,
-            Where=cs.NAME == "VERSION",
+            Where=cs.NAME == Parameter('name'),
         )
 
     @inlineCallbacks
-    def schemaVersion(self):
-        result = yield self._schemaVersion.on(self)
+    def calendarserverValue(self, key):
+        result = yield self._calendarserver.on(self, name=key)
         if result and len(result) == 1:
-            try:
-                returnValue(int(result[0][0]))
-            except ValueError:
-                pass
-        raise RuntimeError("Database schema version cannot be determined.")
+            returnValue(result[0][0])
+        raise RuntimeError("Database key %s cannot be determined." % (key,))
         
 
     @memoizedKey('uid', '_calendarHomes')

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql	2011-11-05 03:51:02 UTC (rev 8258)
@@ -29,7 +29,8 @@
 
 create table CALENDAR_HOME (
   RESOURCE_ID      integer      primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
-  OWNER_UID        varchar(255) not null unique                                 -- implicit index
+  OWNER_UID        varchar(255) not null unique,                                 -- implicit index
+  DATAVERSION	   integer      default 0 not null
 );
 
 ----------------------------
@@ -446,4 +447,4 @@
 );
 
 insert into CALENDARSERVER values ('VERSION', '6');
-
+insert into CALENDARSERVER values ('DATAVERSION', '2');

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql	2011-11-05 03:51:02 UTC (rev 8258)
@@ -18,6 +18,10 @@
 -- Upgrade database schema from VERSION 5 to 6 --
 -------------------------------------------------
 
+-- Just need to add one column
+alter table CALENDAR_HOME
+ add ("DATAVERSION" integer default 1 null);
+ 
 -- Just need to modify one column
 alter table CALENDAR_OBJECT
  add ("SUPPORTED_COMPONENTS" nvarchar2(255) default null);
@@ -25,3 +29,5 @@
 -- Now update the version
 update CALENDARSERVER set VALUE = '6' where NAME = 'VERSION';
 
+-- Also insert the initial data version which we will use in the data upgrade
+insert into CALENDARSERVER values ('DATAVERSION', '1');

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql	2011-11-05 03:51:02 UTC (rev 8258)
@@ -19,9 +19,15 @@
 -------------------------------------------------
 
 -- Just need to add one column
+alter table CALENDAR_HOME
+ add column DATAVERSION integer default 1 null;
+ 
+-- Just need to add one column
 alter table CALENDAR
  add column SUPPORTED_COMPONENTS varchar(255) default null;
 
 -- Now update the version
 update CALENDARSERVER set VALUE = '6' where NAME = 'VERSION';
 
+-- Also insert the initial data version which we will use in the data upgrade
+insert into CALENDARSERVER values ('DATAVERSION', '1');

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -24,7 +24,8 @@
 from twisted.python.modules import getModule
 from twisted.trial.unittest import TestCase
 from txdav.common.datastore.test.util import theStoreBuilder, StubNotifierFactory
-from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService
+from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService,\
+    UpgradeDatabaseDataService
 import re
 
 class SchemaUpgradeTests(TestCase):
@@ -32,11 +33,14 @@
     Tests for L{UpgradeDatabaseSchemaService}.
     """
 
-    def _getSchemaVersion(self, fp):
+    def _getSchemaVersion(self, fp, versionKey):
         schema = fp.getContent()
-        found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", schema)
+        found = re.search("insert into CALENDARSERVER values \('%s', '(\d)+'\);" % (versionKey,), schema)
         if found is None:
-            self.fail("Could not determine schema version for: %s" % (fp,))
+            if versionKey == "VERSION":
+                self.fail("Could not determine schema version for: %s" % (fp,))
+            else:
+                return 1
         return int(found.group(1))
 
     def test_scanUpgradeFiles(self):
@@ -101,28 +105,28 @@
             upgrader = UpgradeDatabaseSchemaService(None, None)
             files = upgrader.scanForUpgradeFiles(dialect)
 
-            current_version = self._getSchemaVersion(upgrader.schemaLocation.child("current.sql"))
+            current_version = self._getSchemaVersion(upgrader.schemaLocation.child("current.sql"), "VERSION")
             
             for child in upgrader.schemaLocation.child("old").globChildren("*.sql"):
-                old_version = self._getSchemaVersion(child)
+                old_version = self._getSchemaVersion(child, "VERSION")
                 upgrades = upgrader.determineUpgradeSequence(old_version, current_version, files, dialect)
                 self.assertNotEqual(len(upgrades), 0)
 
-    def test_upgradeDataAvailability(self):
-        """
-        Make sure that each upgrade file has a valid data upgrade file or None.
-        """
-        
-        for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
-            upgrader = UpgradeDatabaseSchemaService(None, None)
-            files = upgrader.scanForUpgradeFiles(dialect)
-            for _ignore_from, _ignore_to, fp in files:
-                result = upgrader.getDataUpgrade(fp)
-                if result is not None:
-                    self.assertIsInstance(result, types.FunctionType)
+#    def test_upgradeDataAvailability(self):
+#        """
+#        Make sure that each upgrade file has a valid data upgrade file or None.
+#        """
+#        
+#        for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
+#            upgrader = UpgradeDatabaseSchemaService(None, None)
+#            files = upgrader.scanForUpgradeFiles(dialect)
+#            for _ignore_from, _ignore_to, fp in files:
+#                result = upgrader.getDataUpgrade(fp)
+#                if result is not None:
+#                    self.assertIsInstance(result, types.FunctionType)
 
     @inlineCallbacks
-    def test_dbUpgrades(self):
+    def test_dbSchemaUpgrades(self):
         """
         This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
         then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
@@ -169,12 +173,75 @@
         self.addCleanup(_cleanupOldSchema)
 
         test_upgrader = UpgradeDatabaseSchemaService(None, None)
-        expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"))
+        expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"), "VERSION")
         for child in test_upgrader.schemaLocation.child("old").globChildren("*.sql"):
             upgrader = UpgradeDatabaseSchemaService(store, None)
             yield _loadOldSchema(child)
-            yield upgrader.doUpgrade()
+            yield upgrader.databaseUpgrade()
             new_version = yield _loadVersion()
             yield _unloadOldSchema()
 
             self.assertEqual(new_version, expected_version)
+
+    @inlineCallbacks
+    def test_dbDataUpgrades(self):
+        """
+        This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
+        then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
+        postgres.
+        """
+
+        store = yield theStoreBuilder.buildStore(
+            self, StubNotifierFactory()
+        )
+
+        @inlineCallbacks
+        def _loadOldData(path, oldVersion):
+            """
+            Use the postgres schema mechanism to do tests under a separate "namespace"
+            in postgres that we can quickly wipe clean afterwards.
+            """
+            startTxn = store.newTransaction("test_dbUpgrades")        
+            yield startTxn.execSQL("create schema test_dbUpgrades;")
+            yield startTxn.execSQL("set search_path to test_dbUpgrades;")
+            yield startTxn.execSQL(path.getContent())
+            yield startTxn.execSQL("update CALENDARSERVER set VALUE = '%s' where NAME = 'DATAVERSION';" % (oldVersion,))
+            yield startTxn.commit()
+
+        @inlineCallbacks
+        def _loadVersion():
+            startTxn = store.newTransaction("test_dbUpgrades")        
+            new_version = yield startTxn.execSQL("select value from calendarserver where name = 'DATAVERSION';")
+            yield startTxn.commit()
+            returnValue(int(new_version[0][0]))
+
+        @inlineCallbacks
+        def _unloadOldData():
+            startTxn = store.newTransaction("test_dbUpgrades")        
+            yield startTxn.execSQL("set search_path to public;")
+            yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
+            yield startTxn.commit()
+
+        @inlineCallbacks
+        def _cleanupOldData():
+            startTxn = store.newTransaction("test_dbUpgrades")        
+            yield startTxn.execSQL("set search_path to public;")
+            yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade;")
+            yield startTxn.commit()
+
+        self.addCleanup(_cleanupOldData)
+
+        test_upgrader = UpgradeDatabaseSchemaService(None, None)
+        expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"), "DATAVERSION")
+        versions = set()
+        for child in test_upgrader.schemaLocation.child("old").globChildren("*.sql"):
+            versions.add(self._getSchemaVersion(child, "DATAVERSION"))
+
+        for oldVersion in sorted(versions):
+            upgrader = UpgradeDatabaseDataService(store, None)
+            yield _loadOldData(test_upgrader.schemaLocation.child("current.sql"), oldVersion)
+            yield upgrader.databaseUpgrade()
+            new_version = yield _loadVersion()
+            yield _unloadOldData()
+
+            self.assertEqual(new_version, expected_version)

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##
-from twisted.python.reflect import namedObject
 
 """
 Utilities, mostly related to upgrading, common to calendar and addresbook
@@ -24,16 +23,16 @@
 import re
 
 from twext.python.log import LoggingMixIn
+
 from twisted.application.service import Service
 from twisted.internet import reactor
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.python.modules import getModule
+from twisted.python.reflect import namedObject
 
-class UpgradeDatabaseSchemaService(Service, LoggingMixIn, object):
+class UpgradeDatabaseCoreService(Service, LoggingMixIn, object):
     """
-    Checks and upgrades the database schema. This assumes there are a bunch of
-    upgrade files in sql syntax that we can execute against the database to accomplish
-    the upgrade.
+    Base class for either schema or data upgrades on the database.
     """
 
     @classmethod
@@ -58,7 +57,6 @@
         """
         return cls(store, service, uid=uid, gid=gid,)
 
-
     def __init__(self, sqlStore, service, uid=None, gid=None):
         """
         Initialize the service.
@@ -72,66 +70,90 @@
         self.gid = gid
         self.schemaLocation = getModule(__name__).filePath.parent().parent().sibling("sql_schema")
         self.pyLocation = getModule(__name__).filePath.parent()
+        
+        self.versionKey = None
+        self.versionDescriptor = ""
+        self.upgradeFileSuffix = ""
+        self.defaultKeyValue = None
+        
+    def startService(self):
+        """
+        Start the service.
+        """
+        self.databaseUpgrade()
 
     @inlineCallbacks
-    def doUpgrade(self):
+    def databaseUpgrade(self):
         """
-        Do the schema check and upgrade if needed.  Called by C{startService}, but a different method
-        because C{startService} should return C{None}, not a L{Deferred}.
+        Do a database schema upgrade.
+        """
+        self.log_warn("Beginning database %s check." % (self.versionDescriptor,))
+        
+        # Retrieve information from schema and database
+        dialect, required_version, actual_version = yield self.getVersions()
 
-        @return: a Deferred which fires when the migration is complete.
+        if required_version == actual_version:
+            self.log_warn("%s version check complete: no upgrade needed." % (self.versionDescriptor.capitalize(),))
+        elif required_version < actual_version:
+            msg = "Actual %s version %s is more recent than the expected version %s. The service cannot be started" % (
+                self.versionDescriptor, actual_version, required_version,
+            )
+            self.log_error(msg)
+            raise RuntimeError(msg)
+        else:
+            yield self.upgradeVersion(actual_version, required_version, dialect)
+            
+        self.log_warn("Database %s check complete." % (self.versionDescriptor,))
+
+        # see http://twistedmatrix.com/trac/ticket/4649
+        if self.wrappedService is not None:
+            reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
+    
+    @inlineCallbacks
+    def getVersions(self):
         """
-        self.log_warn("Beginning database schema check.")
-        
+        Extract the expected version from the database schema and get the actual version in the current
+        database, along with the DB dialect.
+        """
+
         # Retrieve the version number from the schema file
         current_schema = self.schemaLocation.child("current.sql").getContent()
-        found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", current_schema)
+        found = re.search("insert into CALENDARSERVER values \('%s', '(\d)+'\);" % (self.versionKey,), current_schema)
         if found is None:
-            msg = "Schema is missing required schema VERSION insert statement: %s" % (current_schema,)
+            msg = "Schema is missing required database key %s insert statement: %s" % (self.versionKey, current_schema,)
             self.log_error(msg)
             raise RuntimeError(msg)
         else:
             required_version = int(found.group(1))
-            self.log_warn("Required schema version: %s." % (required_version,))
+            self.log_warn("Required database key %s: %s." % (self.versionKey, required_version,))
         
         # Get the schema version in the current database
         sqlTxn = self.sqlStore.newTransaction()
         dialect = sqlTxn.dialect
         try:
-            actual_version = yield sqlTxn.schemaVersion()
+            actual_version = yield sqlTxn.calendarserverValue(self.versionKey)
+            actual_version = int(actual_version)
             yield sqlTxn.commit()
-        except RuntimeError:
-            self.log_error("Database schema version cannot be determined.")
+        except RuntimeError, ValueError:
+            self.log_error("Database key %s cannot be determined." % (self.versionKey,))
             yield sqlTxn.abort()
-            raise
+            if self.defaultKeyValue is None:
+                raise
+            else:
+                actual_version = self.defaultKeyValue
 
-        self.log_warn("Actual schema version: %s." % (actual_version,))
+        self.log_warn("Actual database key %s: %s." % (self.versionKey, actual_version,))
 
-        if required_version == actual_version:
-            self.log_warn("Schema version check complete: no upgrade needed.")
-        elif required_version < actual_version:
-            msg = "Actual schema version %s is more recent than the expected version %s. The service cannot be started" % (actual_version, required_version,)
-            self.log_error(msg)
-            raise RuntimeError(msg)
-        else:
-            yield self.upgradeVersion(actual_version, required_version, dialect)
-            
-        self.log_warn(
-            "Database schema check complete, launching database service."
-        )
-        # see http://twistedmatrix.com/trac/ticket/4649
-        if self.wrappedService is not None:
-            reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
+        returnValue((dialect, required_version, actual_version,))
 
     @inlineCallbacks
-
     def upgradeVersion(self, fromVersion, toVersion, dialect):
         """
         Update the database from one version to another (the current one). Do this by
         looking for upgrade_from_X_to_Y.sql files that cover the full range of upgrades.
         """
 
-        self.log_warn("Starting schema upgrade from version %d to %d." % (fromVersion, toVersion,))
+        self.log_warn("Starting %s upgrade from version %d to %d." % (self.versionDescriptor, fromVersion, toVersion,))
         
         # Scan for all possible upgrade files - returned sorted
         files = self.scanForUpgradeFiles(dialect)
@@ -140,27 +162,30 @@
         upgrades = self.determineUpgradeSequence(fromVersion, toVersion, files, dialect)
 
         # Use one transaction for the entire set of upgrades
-        sqlTxn = self.sqlStore.newTransaction()
         try:
             for fp in upgrades:
-                yield self.applyUpgrade(sqlTxn, fp)
-            yield sqlTxn.commit()
+                yield self.applyUpgrade(fp)
         except RuntimeError:
-            self.log_error("Database upgrade failed:" % (fp.basename(),))
-            yield sqlTxn.abort()
+            self.log_error("Database %s upgrade failed using: %s" % (self.versionDescriptor, fp.basename(),))
             raise
 
-        self.log_warn("Schema upgraded from version %d to %d." % (fromVersion, toVersion,))
+        self.log_warn("%s upgraded from version %d to %d." % (self.versionDescriptor.capitalize(), fromVersion, toVersion,))
 
+    def getPathToUpgrades(self, dialect):
+        """
+        Return the path where appropriate upgrade files can be found. 
+        """
+        raise NotImplementedError
+
     def scanForUpgradeFiles(self, dialect):
         """
-        Scan the module path for upgrade files with the require name.
+        Scan for upgrade files with the require name.
         """
         
-        fp = self.schemaLocation.child("upgrades").child(dialect)
+        fp = self.getPathToUpgrades(dialect)
         upgrades = []
-        regex = re.compile("upgrade_from_(\d)+_to_(\d)+.sql")
-        for child in fp.globChildren("upgrade_*.sql"):
+        regex = re.compile("upgrade_from_(\d)+_to_(\d)+%s" % (self.upgradeFileSuffix,))
+        for child in fp.globChildren("upgrade_*%s" % (self.upgradeFileSuffix,)):
             matched = regex.match(child.basename())
             if matched is not None:
                 fromV = int(matched.group(1))
@@ -169,10 +194,10 @@
         
         upgrades.sort(key=lambda x:(x[0], x[1]))
         return upgrades
-    
+
     def determineUpgradeSequence(self, fromVersion, toVersion, files, dialect):
         """
-        Determine the upgrade_from_X_to_Y.sql files that cover the full range of upgrades.
+        Determine the upgrade_from_X_to_Y(.sql|.py) files that cover the full range of upgrades.
         Note that X and Y may not be consecutive, e.g., we might have an upgrade from 3 to 4,
         4 to 5, and 3 to 5 - the later because it is more efficient to jump over the intermediate
         step. As a result we will always try and pick the upgrade file that gives the biggest
@@ -198,39 +223,88 @@
         
         return upgrades
 
+    def applyUpgrade(self, fp):
+        """
+        Apply the supplied upgrade to the database. Always return an L{Deferred"
+        """
+        raise NotImplementedError
+        
+class UpgradeDatabaseSchemaService(UpgradeDatabaseCoreService):
+    """
+    Checks and upgrades the database schema. This assumes there are a bunch of
+    upgrade files in sql syntax that we can execute against the database to accomplish
+    the upgrade.
+    """
+
+    def __init__(self, sqlStore, service, uid=None, gid=None):
+        """
+        Initialize the service.
+        
+        @param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
+        @param service:  Wrapped service. Can be C{None} when doing unit tests.
+        """
+        super(UpgradeDatabaseSchemaService, self).__init__(sqlStore, service, uid, gid)
+        
+        self.versionKey = "VERSION"
+        self.versionDescriptor = "schema"
+        self.upgradeFileSuffix = ".sql"
+
+    def getPathToUpgrades(self, dialect):
+        return self.schemaLocation.child("upgrades").child(dialect)
+
     @inlineCallbacks
-    def applyUpgrade(self, sqlTxn, fp):
+    def applyUpgrade(self, fp):
         """
         Apply the schema upgrade .sql file to the database.
         """
         self.log_warn("Applying schema upgrade: %s" % (fp.basename(),))
-        sql = fp.getContent()
-        yield sqlTxn.execSQLBlock(sql)
-        
-        doDataUpgrade = self.getDataUpgrade(fp)
-        if doDataUpgrade is not None:
-            yield doDataUpgrade(sqlTxn)
+        sqlTxn = self.sqlStore.newTransaction()
+        try:
+            sql = fp.getContent()
+            yield sqlTxn.execSQLBlock(sql)
+            yield sqlTxn.commit()
+        except RuntimeError:
+            yield sqlTxn.abort()
+            raise
 
-    def getDataUpgrade(self, fp):        
-        # Also look for python module to execute
-        check_name = self.pyLocation.child(fp.basename()[:-4] + ".py")
-        if check_name.exists():
-            try:
-                module = getModule(__name__)
-                module = ".".join(module.name.split(".")[:-1]) + "." + fp.basename()[:-4] + ".doUpgrade"
-                doUpgrade = namedObject(module)
-                self.log_warn("Applying data upgrade: %s" % (module,))
-                return doUpgrade
-            except ImportError:
-                msg = "Failed data upgrade: %s" % (fp.basename()[:-4],)
-                self.log_error(msg)
-                raise RuntimeError(msg)
-        else:
-            self.log_warn("No data upgrade: %s" % (fp.basename()[:-4],))
-            return None
+class UpgradeDatabaseDataService(UpgradeDatabaseCoreService):
+    """
+    Checks and upgrades the database data. This assumes there are a bunch of
+    upgrade python modules that we can execute against the database to accomplish
+    the upgrade.
+    """
+
+    def __init__(self, sqlStore, service, uid=None, gid=None):
+        """
+        Initialize the service.
         
-    def startService(self):
+        @param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
+        @param service:  Wrapped service. Can be C{None} when doing unit tests.
         """
-        Start the service.
+        super(UpgradeDatabaseDataService, self).__init__(sqlStore, service, uid, gid)
+        
+        self.versionKey = "DATAVERSION"
+        self.versionDescriptor = "data"
+        self.upgradeFileSuffix = ".py"
+
+    def getPathToUpgrades(self, dialect):
+        return self.pyLocation.child("upgrades")
+
+    @inlineCallbacks
+    def applyUpgrade(self, fp):
         """
-        self.doUpgrade()
+        Apply the data upgrade .py files to the database.
+        """
+        
+        # Find the module function we need to execute
+        try:
+            module = getModule(__name__)
+            module = ".".join(module.name.split(".")[:-1]) + ".upgrades." + fp.basename()[:-3] + ".doUpgrade"
+            doUpgrade = namedObject(module)
+        except ImportError:
+            msg = "Failed data upgrade: %s" % (fp.basename()[:-4],)
+            self.log_error(msg)
+            raise RuntimeError(msg)
+
+        self.log_warn("Applying data upgrade: %s" % (module,))
+        yield doUpgrade(self.sqlStore)

Deleted: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -1,50 +0,0 @@
-# -*- test-case-name: txdav.common.datastore.upgrade.sql.test -*-
-##
-# Copyright (c) 2011 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from twext.enterprise.dal.syntax import Update
-from twext.web2.dav.element.parser import WebDAVDocument
-from twisted.internet.defer import inlineCallbacks
-from twistedcaldav import caldavxml
-from txdav.common.datastore.sql_tables import schema
-from txdav.common.datastore.upgrade.sql.util import rowsForProperty,\
-    removeProperty
-
-"""
-Data upgrade from database version 5 to 6
-"""
-
- at inlineCallbacks
-def doUpgrade(sqlTxn):
-    """
-    Need to move all the CalDAV:supported-component-set properties in the RESOURCE_PROPERTY
-    table to the new CALENDAR table column, extracting the new format value from the XML property.
-    """
-
-    rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
-    for calendar_rid, value in rows:
-        prop = WebDAVDocument.fromString(value).root_element
-        supported_components = ",".join(sorted([comp.attributes["name"].upper() for comp in prop.children]))
-
-        cal = schema.CALENDAR
-        yield Update(
-            {
-                cal.SUPPORTED_COMPONENTS : supported_components
-            },
-            Where=(cal.RESOURCE_ID == calendar_rid)
-        ).on(sqlTxn)
-
-    yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)

Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/__init__.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/__init__.py	                        (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/__init__.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -0,0 +1,16 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+

Copied: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_1_to_2.py (from rev 8190, CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_1_to_2.py	                        (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_1_to_2.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -0,0 +1,85 @@
+# -*- test-case-name: txdav.common.datastore.upgrade.sql.test -*-
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.syntax import Update
+from twext.web2.dav.element.parser import WebDAVDocument
+from twisted.internet.defer import inlineCallbacks
+from twistedcaldav import caldavxml
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.upgrade.sql.upgrades.util import rowsForProperty,\
+    removeProperty, updateDataVersion, doToEachCalendarHomeNotAtVersion
+
+"""
+Data upgrade from database version 1 to 2
+"""
+
+UPGRADE_TO_VERSION = 2
+
+ at inlineCallbacks
+def doUpgrade(sqlStore):
+    """
+    Do the required upgrade steps.
+    """
+    yield moveSupportedComponentSetProperties(sqlStore)
+    yield splitCalendars(sqlStore)
+    
+    # Always bump the DB value
+    yield updateDataVersion(sqlStore, UPGRADE_TO_VERSION)
+
+ at inlineCallbacks
+def moveSupportedComponentSetProperties(sqlStore):
+    """
+    Need to move all the CalDAV:supported-component-set properties in the RESOURCE_PROPERTY
+    table to the new CALENDAR table column, extracting the new format value from the XML property.
+    """
+
+    sqlTxn = sqlStore.newTransaction()
+    try:
+        rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
+        for calendar_rid, value in rows:
+            prop = WebDAVDocument.fromString(value).root_element
+            supported_components = ",".join(sorted([comp.attributes["name"].upper() for comp in prop.children]))
+    
+            cal = schema.CALENDAR
+            yield Update(
+                {
+                    cal.SUPPORTED_COMPONENTS : supported_components
+                },
+                Where=(cal.RESOURCE_ID == calendar_rid)
+            ).on(sqlTxn)
+    
+        yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)
+        yield sqlTxn.commit()
+    except RuntimeError:
+        yield sqlTxn.abort()
+        raise
+
+ at inlineCallbacks
+def splitCalendars(sqlStore):
+    """
+    Split all calendars by component type.
+    """
+
+    @inlineCallbacks
+    def doIt(home):
+        """
+        Split each regular calendar in the home.
+        """
+        yield home.splitCalendars()
+            
+    # Do this to each calendar home not already at version 2
+    yield doToEachCalendarHomeNotAtVersion(sqlStore, UPGRADE_TO_VERSION, doIt)

Copied: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/util.py (from rev 8190, CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/util.py	                        (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrades/util.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -0,0 +1,93 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.syntax import Select, Delete, Update
+from twisted.internet.defer import inlineCallbacks, returnValue
+from txdav.base.propertystore.base import PropertyName
+from txdav.common.datastore.sql_tables import schema
+
+ at inlineCallbacks
+def rowsForProperty(txn, propelement):
+    pname = PropertyName.fromElement(propelement)
+
+    rp = schema.RESOURCE_PROPERTY
+    rows = yield Select(
+        [rp.RESOURCE_ID, rp.VALUE,],
+        From=rp,
+        Where=rp.NAME == pname.toString(),
+    ).on(txn)
+    
+    returnValue(rows)
+
+ at inlineCallbacks
+def removeProperty(txn, propelement):
+    pname = PropertyName.fromElement(propelement)
+
+    rp = schema.RESOURCE_PROPERTY
+    yield Delete(
+        From=rp,
+        Where=rp.NAME == pname.toString(),
+    ).on(txn)
+
+ at inlineCallbacks
+def updateDataVersion(store, version):
+
+    txn = store.newTransaction("updateDataVersion")    
+    cs = schema.CALENDARSERVER
+    yield Update(
+        {cs.VALUE: version},
+        Where=cs.NAME == "DATAVERSION",
+    ).on(txn)
+    yield txn.commit()
+
+ at inlineCallbacks
+def doToEachCalendarHomeNotAtVersion(store, version, doIt):
+    """
+    Do something to each calendar home whose version column indicates it is older
+    than the specified version. Do this in batches as there may be a lot of work to do.
+    """
+
+    while True:
+        
+        # Get the next home with an old version
+        txn = store.newTransaction("updateDataVersion")   
+        try: 
+            ch = schema.CALENDAR_HOME
+            rows = yield Select(
+                [ch.RESOURCE_ID, ch.OWNER_UID,],
+                From=ch,
+                Where=ch.DATAVERSION < version,
+                OrderBy=ch.OWNER_UID,
+                Limit=1,
+            ).on(txn)
+            
+            if len(rows) == 0:
+                yield txn.commit()
+                returnValue(None)
+            
+            # Apply to the home
+            resource_id, _ignore_owner_uid = rows[0]
+            home = yield txn.calendarHomeWithResourceID(resource_id)
+            yield doIt(home)
+    
+            # Update the home to the current version
+            yield Update(
+                {ch.DATAVERSION: version},
+                Where=ch.RESOURCE_ID == resource_id,
+            ).on(txn)
+            yield txn.commit()
+        except RuntimeError:
+            yield txn.abort()

Deleted: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -1,43 +0,0 @@
-##
-# Copyright (c) 2011 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from twext.enterprise.dal.syntax import Select, Delete
-from twisted.internet.defer import inlineCallbacks, returnValue
-from txdav.base.propertystore.base import PropertyName
-from txdav.common.datastore.sql_tables import schema
-
- at inlineCallbacks
-def rowsForProperty(txn, propelement):
-    pname = PropertyName.fromElement(propelement)
-
-    rp = schema.RESOURCE_PROPERTY
-    rows = yield Select(
-        [rp.RESOURCE_ID, rp.VALUE,],
-        From=rp,
-        Where=rp.NAME == pname.toString(),
-    ).on(txn)
-    
-    returnValue(rows)
-
- at inlineCallbacks
-def removeProperty(txn, propelement):
-    pname = PropertyName.fromElement(propelement)
-
-    rp = schema.RESOURCE_PROPERTY
-    yield Delete(
-        From=rp,
-        Where=rp.NAME == pname.toString(),
-    ).on(txn)

Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py	2011-11-04 16:43:30 UTC (rev 8257)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py	2011-11-05 03:51:02 UTC (rev 8258)
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##
-from twistedcaldav.config import config
 
 """
 Tests for L{txdav.common.datastore.upgrade.migrate}.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20111104/ac78ed74/attachment-0001.html>


More information about the calendarserver-changes mailing list