[CalendarServer-changes] [3688] CalendarServer/branches/users/sagen/migration-3686

source_changes at macosforge.org source_changes at macosforge.org
Thu Feb 19 12:45:15 PST 2009


Revision: 3688
          http://trac.macosforge.org/projects/calendarserver/changeset/3688
Author:   sagen at apple.com
Date:     2009-02-19 12:45:12 -0800 (Thu, 19 Feb 2009)
Log Message:
-----------
Checking in work in progress for upgrade/migration branch

Modified Paths:
--------------
    CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py
    CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py
    CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py

Modified: CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py	2009-02-19 20:00:23 UTC (rev 3687)
+++ CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py	2009-02-19 20:45:12 UTC (rev 3688)
@@ -324,8 +324,6 @@
 
     def makeService(self, options):
 
-        # Now do any on disk upgrades we might need.
-        UpgradeTheServer.doUpgrade()
 
         serviceMethod = getattr(self, "makeService_%s" % (config.ProcessType,), None)
 
@@ -336,6 +334,16 @@
                 % (config.ProcessType,)
             )
         else:
+
+            if config.ProcessType in ('Combined', 'Single'):
+
+                # Process localization string files
+                processLocalizationFiles(config.Localization)
+
+                # Now do any on disk upgrades we might need.
+                UpgradeTheServer.doUpgrade(config)
+
+
             service = serviceMethod(options)
 
             #

Modified: CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py
===================================================================
--- CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py	2009-02-19 20:00:23 UTC (rev 3687)
+++ CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py	2009-02-19 20:45:12 UTC (rev 3688)
@@ -44,7 +44,7 @@
     BaseDirectoryService = namedClass(config.DirectoryService.type)
 
     class MyDirectoryService (BaseDirectoryService):
-        def principalCollection(self):
+        def getPrincipalCollection(self):
             if not hasattr(self, "_principalCollection"):
                 #
                 # Instantiating a CalendarHomeProvisioningResource with a directory
@@ -60,12 +60,26 @@
 
             return self._principalCollection
 
+        def setPrincipalCollection(self, coll):
+            # See principal.py line 237:  self.directory.principalCollection = self
+            pass
+
+        principalCollection = property(getPrincipalCollection, setPrincipalCollection)
+
         def calendarHomeForShortName(self, recordType, shortName):
-            principal = self.principalCollection().principalForShortName(recordType, shortName)
+            principal = self.principalCollection.principalForShortName(recordType, shortName)
             if principal:
                 return principal.calendarHome()
             return None
 
+        def principalForCalendarUserAddress(self, cua):
+            record = self.recordWithCalendarUserAddress(cua)
+            if record is not None:
+                return self.principalCollection.principalForUID(record.uid)
+            else:
+                return None
+
+
     return MyDirectoryService(**config.DirectoryService.params)
 
 class DummyDirectoryService (DirectoryService):

Modified: CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py	2009-02-19 20:00:23 UTC (rev 3687)
+++ CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py	2009-02-19 20:45:12 UTC (rev 3688)
@@ -14,29 +14,258 @@
 # limitations under the License.
 ##
 
+from __future__ import with_statement
+
 from twisted.web2.dav.fileop import rmdir
-from twistedcaldav.config import config
+from twistedcaldav.directory.directory import DirectoryService
 from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
 from twistedcaldav.log import Logger
-import os
+from twistedcaldav.ical import Component
+from twistedcaldav.scheduling.cuaddress import normalizeCUAddr
+from calendarserver.tools.util import getDirectory, dummyDirectoryRecord
+import xattr, itertools, os
 
 log = Logger()
 
+
+def upgrade_to_1(config):
+    log.info("Upgrading to 1")
+
+
+    def fixBadQuotes(data):
+        if (
+            data.find('\\"') != -1 or
+            data.find('\\\r\n "') != -1 or
+            data.find('\r\n \r\n "') != -1
+        ):
+            # Fix by continuously replacing \" with " until no more
+            # replacements occur
+            while True:
+                newData = data.replace('\\"', '"').replace('\\\r\n "', '\r\n "').replace('\r\n \r\n "', '\r\n "')
+                if newData == data:
+                    break
+                else:
+                    data = newData
+
+            return data, True
+        else:
+            return data, False
+
+
+
+    def normalizeCUAddrs(data, directory):
+        cal = Component.fromString(data)
+        print cal
+        for component in cal.subcomponents():
+            if component.name() != "VTIMEZONE":
+                for prop in itertools.chain(
+                    component.properties("ORGANIZER"),
+                    component.properties("ATTENDEE"),
+                ):
+                    cua = normalizeCUAddr(prop.value())
+                    try:
+                        principal = directory.principalForCalendarUserAddress(cua)
+                    except Exception, e:
+                        # lookup failed
+                        log.debug("Lookup of %s failed: %s" % (cua, e))
+                        principal = None
+
+                    if principal is not None:
+                        prop.setValue("urn:uuid:%s" % (principal.record.guid,))
+                        if principal.record.fullName:
+                            prop.params()["CN"] = [principal.record.fullName,]
+                        else:
+                            try:
+                                del prop.params()["CN"]
+                            except KeyError:
+                                pass
+
+                        # Re-write the X-CALENDARSERVER-EMAIL if its value no longer matches
+                        oldEmail = prop.params().get("X-CALENDARSERVER-EMAIL", (None,))[0]
+                        if oldEmail:
+                            oldEmail = "mailto:%s" % (oldEmail,)
+
+                        if oldEmail is None or oldEmail not in principal.record.calendarUserAddresses:
+                            if cua.startswith("mailto:") and cua in principal.record.calendarUserAddresses:
+                                email = cua[7:]
+                            else:
+                                for addr in principal.record.calendarUserAddresses:
+                                    if addr.startswith("mailto:"):
+                                        email = addr[7:]
+                                        break
+                                else:
+                                    email = None
+
+                            if email:
+                                prop.params()["X-CALENDARSERVER-EMAIL"] = [email,]
+                            else:
+                                try:
+                                    del prop.params()["X-CALENDARSERVER-EMAIL"]
+                                except KeyError:
+                                    pass
+
+        newData = str(cal)
+        return newData, not newData == data
+
+
+    def upgradeCalendarCollection(oldCal, newCal, directory):
+        os.rename(oldCal, newCal)
+
+        for resource in os.listdir(newCal):
+            resourcePath = os.path.join(newCal, resource)
+
+            # MOR:  Is this the proper way to tell we have a caldav resource?
+            # (".ics" extension-checking doesn't seem safe)
+            xattrs = xattr.xattr(resourcePath)
+            if not xattrs.has_key("WebDAV:{DAV:}getcontenttype"):
+                continue
+
+            log.info("Processing: %s" % (resourcePath,))
+            needsRewrite = False
+            with open(resourcePath) as res:
+                data = res.read()
+
+                data, fixed = fixBadQuotes(data)
+                if fixed:
+                    needsRewrite = True
+                    log.info("Fixing bad quotes in %s" % (resourcePath,))
+
+                data, fixed = normalizeCUAddrs(data, directory)
+                if fixed:
+                    needsRewrite = True
+                    log.info("Normalized CUAddrs in %s" % (resourcePath,))
+                    print "NORMALIZED TO:\n%s" % (data,)
+
+            if needsRewrite:
+                with open(resourcePath, "w") as res:
+                    res.write(data)
+
+
+    def upgradeCalendarHome(oldHome, newHome, directory):
+        try:
+            os.makedirs(newHome)
+        except:
+            log.info("Skipping upgrade of %s because %s already exists" %
+                (oldHome, newHome))
+            return
+
+        log.info("Upgrading calendar home: %s -> %s" % (oldHome, newHome))
+
+        for cal in os.listdir(oldHome):
+            oldCal = os.path.join(oldHome, cal)
+
+            # xattrs = xattr.xattr(oldCal)
+            # if not xattrs.has_key("WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"):
+            #     continue
+
+            newCal = os.path.join(newHome, cal)
+            log.info("Upgrading calendar: %s" % (newCal,))
+            upgradeCalendarCollection(oldCal, newCal, directory)
+
+        os.rmdir(oldHome)
+
+
+    docRoot = config.DocumentRoot
+    # MOR: Temporary:
+    docRoot = "/Users/morgen/Migration/CalendarServer/Documents"
+
+    directory = getDirectory()
+
+    if os.path.exists(docRoot):
+
+        calRoot = os.path.join(docRoot, "calendars")
+        if os.path.exists(calRoot):
+
+            uidHomes = os.path.join(calRoot, "__uids__")
+
+            if os.path.exists(uidHomes):
+                for home in os.listdir(uidHomes):
+
+                    # MOR: This assumes no UID is going to be 2 chars or less
+                    if len(home) <= 2:
+                        continue
+
+                    oldHome = os.path.join(uidHomes, home)
+                    newHome = os.path.join(uidHomes, home[0:2], home[2:4], home)
+                    upgradeCalendarHome(oldHome, newHome, directory)
+
+            else:
+                os.mkdir(uidHomes)
+
+            for recordType, dirName in (
+                (DirectoryService.recordType_users, "users"),
+                (DirectoryService.recordType_groups, "groups"),
+                (DirectoryService.recordType_locations, "locations"),
+                (DirectoryService.recordType_resources, "resources"),
+            ):
+                dirPath = os.path.join(calRoot, dirName)
+                if os.path.exists(dirPath):
+                    for shortName in os.listdir(dirPath):
+                        record = directory.recordWithShortName(recordType,
+                            shortName)
+                        if record is not None:
+                            uid = record.uid
+                            oldHome = os.path.join(dirPath, shortName)
+                            newHome = os.path.join(uidHomes, uid[0:2], uid[2:4],
+                                uid)
+                            upgradeCalendarHome(oldHome, newHome, directory)
+
+
+
+    
+    # Don't forget to also do this:
+
+    # UpgradeTheServer._doPrincipalCollectionInMemoryUpgrade(config)
+
 class UpgradeTheServer(object):
+
+    # Each method in this array will upgrade from one version to the next;
+    # the index of each method within the array corresponds to the on-disk
+    # version number that it upgrades from.  For example, if the on-disk
+    # .version file contains a "3", but there are 6 methods in this array,
+    # methods 3 through 5 (using 0-based array indexing) will be executed in
+    # order.
+    upgradeMethods = [
+        upgradeLeopardToSnowLeopard,
+    ]
     
     @staticmethod
-    def doUpgrade():
+    def doUpgrade(config):
         
-        UpgradeTheServer._doPrincipalCollectionInMemoryUpgrade()
+        # import pdb; pdb.set_trace()
+
+        docRoot = config.DocumentRoot
+        # MOR: Temporary:
+        docRoot = "/Users/morgen/Migration/CalendarServer/Documents"
+        versionFilePath = os.path.join(docRoot, ".calendarserver_version")
+
+        newestVersion = len(UpgradeTheServer.upgradeMethods)
+
+        onDiskVersion = 0
+        if os.path.exists(versionFilePath):
+            try:
+                with open(versionFilePath) as versionFile:
+                    onDiskVersion = int(versionFile.read().strip())
+            except IOError, e:
+                log.error("Cannot open %s; skipping migration" %
+                    (versionFilePath,))
+            except ValueError, e:
+                log.error("Invalid version number in %s; skipping migration" %
+                    (versionFilePath,))
+
+        for upgradeVersion in range(onDiskVersion, newestVersion):
+            UpgradeTheServer.upgradeMethods[upgradeVersion](config)
+
     
+
     @staticmethod
-    def _doPrincipalCollectionInMemoryUpgrade():
+    def _doPrincipalCollectionInMemoryUpgrade(config):
         
         # Look for the /principals/ directory on disk
         old_principals = os.path.join(config.DocumentRoot, "principals")
         if os.path.exists(old_principals):
             # First move the proxy database and rename it
-            UpgradeTheServer._doProxyDatabaseMoveUpgrade()
+            UpgradeTheServer._doProxyDatabaseMoveUpgrade(config)
         
             # Now delete the on disk representation of principals
             rmdir(old_principals)
@@ -46,7 +275,7 @@
             )
 
     @staticmethod
-    def _doProxyDatabaseMoveUpgrade():
+    def _doProxyDatabaseMoveUpgrade(config):
         
         # See if the old DB is present
         old_db_path = os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090219/989f8356/attachment-0001.html>


More information about the calendarserver-changes mailing list