[CalendarServer-changes] [3688] CalendarServer/branches/users/sagen/migration-3686
source_changes at macosforge.org
source_changes at macosforge.org
Thu Feb 19 12:45:15 PST 2009
Revision: 3688
http://trac.macosforge.org/projects/calendarserver/changeset/3688
Author: sagen at apple.com
Date: 2009-02-19 12:45:12 -0800 (Thu, 19 Feb 2009)
Log Message:
-----------
Checking in work in progress for upgrade/migration branch
Modified Paths:
--------------
CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py
CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py
CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py
Modified: CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py 2009-02-19 20:00:23 UTC (rev 3687)
+++ CalendarServer/branches/users/sagen/migration-3686/calendarserver/tap/caldav.py 2009-02-19 20:45:12 UTC (rev 3688)
@@ -324,8 +324,6 @@
def makeService(self, options):
- # Now do any on disk upgrades we might need.
- UpgradeTheServer.doUpgrade()
serviceMethod = getattr(self, "makeService_%s" % (config.ProcessType,), None)
@@ -336,6 +334,16 @@
% (config.ProcessType,)
)
else:
+
+ if config.ProcessType in ('Combined', 'Single'):
+
+ # Process localization string files
+ processLocalizationFiles(config.Localization)
+
+ # Now do any on disk upgrades we might need.
+ UpgradeTheServer.doUpgrade(config)
+
+
service = serviceMethod(options)
#
Modified: CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py
===================================================================
--- CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py 2009-02-19 20:00:23 UTC (rev 3687)
+++ CalendarServer/branches/users/sagen/migration-3686/calendarserver/tools/util.py 2009-02-19 20:45:12 UTC (rev 3688)
@@ -44,7 +44,7 @@
BaseDirectoryService = namedClass(config.DirectoryService.type)
class MyDirectoryService (BaseDirectoryService):
- def principalCollection(self):
+ def getPrincipalCollection(self):
if not hasattr(self, "_principalCollection"):
#
# Instantiating a CalendarHomeProvisioningResource with a directory
@@ -60,12 +60,26 @@
return self._principalCollection
+ def setPrincipalCollection(self, coll):
+ # See principal.py line 237: self.directory.principalCollection = self
+ pass
+
+ principalCollection = property(getPrincipalCollection, setPrincipalCollection)
+
def calendarHomeForShortName(self, recordType, shortName):
- principal = self.principalCollection().principalForShortName(recordType, shortName)
+ principal = self.principalCollection.principalForShortName(recordType, shortName)
if principal:
return principal.calendarHome()
return None
+ def principalForCalendarUserAddress(self, cua):
+ record = self.recordWithCalendarUserAddress(cua)
+ if record is not None:
+ return self.principalCollection.principalForUID(record.uid)
+ else:
+ return None
+
+
return MyDirectoryService(**config.DirectoryService.params)
class DummyDirectoryService (DirectoryService):
Modified: CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py 2009-02-19 20:00:23 UTC (rev 3687)
+++ CalendarServer/branches/users/sagen/migration-3686/twistedcaldav/upgrade.py 2009-02-19 20:45:12 UTC (rev 3688)
@@ -14,29 +14,258 @@
# limitations under the License.
##
+from __future__ import with_statement
+
from twisted.web2.dav.fileop import rmdir
-from twistedcaldav.config import config
+from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
from twistedcaldav.log import Logger
-import os
+from twistedcaldav.ical import Component
+from twistedcaldav.scheduling.cuaddress import normalizeCUAddr
+from calendarserver.tools.util import getDirectory, dummyDirectoryRecord
+import xattr, itertools, os
log = Logger()
+
+def upgrade_to_1(config):
+ log.info("Upgrading to 1")
+
+
+ def fixBadQuotes(data):
+ if (
+ data.find('\\"') != -1 or
+ data.find('\\\r\n "') != -1 or
+ data.find('\r\n \r\n "') != -1
+ ):
+ # Fix by continuously replacing \" with " until no more
+ # replacements occur
+ while True:
+ newData = data.replace('\\"', '"').replace('\\\r\n "', '\r\n "').replace('\r\n \r\n "', '\r\n "')
+ if newData == data:
+ break
+ else:
+ data = newData
+
+ return data, True
+ else:
+ return data, False
+
+
+
+ def normalizeCUAddrs(data, directory):
+ cal = Component.fromString(data)
+ print cal
+ for component in cal.subcomponents():
+ if component.name() != "VTIMEZONE":
+ for prop in itertools.chain(
+ component.properties("ORGANIZER"),
+ component.properties("ATTENDEE"),
+ ):
+ cua = normalizeCUAddr(prop.value())
+ try:
+ principal = directory.principalForCalendarUserAddress(cua)
+ except Exception, e:
+ # lookup failed
+ log.debug("Lookup of %s failed: %s" % (cua, e))
+ principal = None
+
+ if principal is not None:
+ prop.setValue("urn:uuid:%s" % (principal.record.guid,))
+ if principal.record.fullName:
+ prop.params()["CN"] = [principal.record.fullName,]
+ else:
+ try:
+ del prop.params()["CN"]
+ except KeyError:
+ pass
+
+ # Re-write the X-CALENDARSERVER-EMAIL if its value no longer matches
+ oldEmail = prop.params().get("X-CALENDARSERVER-EMAIL", (None,))[0]
+ if oldEmail:
+ oldEmail = "mailto:%s" % (oldEmail,)
+
+ if oldEmail is None or oldEmail not in principal.record.calendarUserAddresses:
+ if cua.startswith("mailto:") and cua in principal.record.calendarUserAddresses:
+ email = cua[7:]
+ else:
+ for addr in principal.record.calendarUserAddresses:
+ if addr.startswith("mailto:"):
+ email = addr[7:]
+ break
+ else:
+ email = None
+
+ if email:
+ prop.params()["X-CALENDARSERVER-EMAIL"] = [email,]
+ else:
+ try:
+ del prop.params()["X-CALENDARSERVER-EMAIL"]
+ except KeyError:
+ pass
+
+ newData = str(cal)
+ return newData, not newData == data
+
+
+ def upgradeCalendarCollection(oldCal, newCal, directory):
+ os.rename(oldCal, newCal)
+
+ for resource in os.listdir(newCal):
+ resourcePath = os.path.join(newCal, resource)
+
+ # MOR: Is this the proper way to tell we have a caldav resource?
+ # (".ics" extension-checking doesn't seem safe)
+ xattrs = xattr.xattr(resourcePath)
+ if not xattrs.has_key("WebDAV:{DAV:}getcontenttype"):
+ continue
+
+ log.info("Processing: %s" % (resourcePath,))
+ needsRewrite = False
+ with open(resourcePath) as res:
+ data = res.read()
+
+ data, fixed = fixBadQuotes(data)
+ if fixed:
+ needsRewrite = True
+ log.info("Fixing bad quotes in %s" % (resourcePath,))
+
+ data, fixed = normalizeCUAddrs(data, directory)
+ if fixed:
+ needsRewrite = True
+ log.info("Normalized CUAddrs in %s" % (resourcePath,))
+ print "NORMALIZED TO:\n%s" % (data,)
+
+ if needsRewrite:
+ with open(resourcePath, "w") as res:
+ res.write(data)
+
+
+ def upgradeCalendarHome(oldHome, newHome, directory):
+ try:
+ os.makedirs(newHome)
+ except:
+ log.info("Skipping upgrade of %s because %s already exists" %
+ (oldHome, newHome))
+ return
+
+ log.info("Upgrading calendar home: %s -> %s" % (oldHome, newHome))
+
+ for cal in os.listdir(oldHome):
+ oldCal = os.path.join(oldHome, cal)
+
+ # xattrs = xattr.xattr(oldCal)
+ # if not xattrs.has_key("WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"):
+ # continue
+
+ newCal = os.path.join(newHome, cal)
+ log.info("Upgrading calendar: %s" % (newCal,))
+ upgradeCalendarCollection(oldCal, newCal, directory)
+
+ os.rmdir(oldHome)
+
+
+ docRoot = config.DocumentRoot
+ # MOR: Temporary:
+ docRoot = "/Users/morgen/Migration/CalendarServer/Documents"
+
+ directory = getDirectory()
+
+ if os.path.exists(docRoot):
+
+ calRoot = os.path.join(docRoot, "calendars")
+ if os.path.exists(calRoot):
+
+ uidHomes = os.path.join(calRoot, "__uids__")
+
+ if os.path.exists(uidHomes):
+ for home in os.listdir(uidHomes):
+
+ # MOR: This assumes no UID is going to be 2 chars or less
+ if len(home) <= 2:
+ continue
+
+ oldHome = os.path.join(uidHomes, home)
+ newHome = os.path.join(uidHomes, home[0:2], home[2:4], home)
+ upgradeCalendarHome(oldHome, newHome, directory)
+
+ else:
+ os.mkdir(uidHomes)
+
+ for recordType, dirName in (
+ (DirectoryService.recordType_users, "users"),
+ (DirectoryService.recordType_groups, "groups"),
+ (DirectoryService.recordType_locations, "locations"),
+ (DirectoryService.recordType_resources, "resources"),
+ ):
+ dirPath = os.path.join(calRoot, dirName)
+ if os.path.exists(dirPath):
+ for shortName in os.listdir(dirPath):
+ record = directory.recordWithShortName(recordType,
+ shortName)
+ if record is not None:
+ uid = record.uid
+ oldHome = os.path.join(dirPath, shortName)
+ newHome = os.path.join(uidHomes, uid[0:2], uid[2:4],
+ uid)
+ upgradeCalendarHome(oldHome, newHome, directory)
+
+
+
+
+ # Don't forget to also do this:
+
+ # UpgradeTheServer._doPrincipalCollectionInMemoryUpgrade(config)
+
class UpgradeTheServer(object):
+
+ # Each method in this array will upgrade from one version to the next;
+ # the index of each method within the array corresponds to the on-disk
+ # version number that it upgrades from. For example, if the on-disk
+ # .version file contains a "3", but there are 6 methods in this array,
+ # methods 3 through 5 (using 0-based array indexing) will be executed in
+ # order.
+ upgradeMethods = [
+ upgradeLeopardToSnowLeopard,
+ ]
@staticmethod
- def doUpgrade():
+ def doUpgrade(config):
- UpgradeTheServer._doPrincipalCollectionInMemoryUpgrade()
+ # import pdb; pdb.set_trace()
+
+ docRoot = config.DocumentRoot
+ # MOR: Temporary:
+ docRoot = "/Users/morgen/Migration/CalendarServer/Documents"
+ versionFilePath = os.path.join(docRoot, ".calendarserver_version")
+
+ newestVersion = len(UpgradeTheServer.upgradeMethods)
+
+ onDiskVersion = 0
+ if os.path.exists(versionFilePath):
+ try:
+ with open(versionFilePath) as versionFile:
+ onDiskVersion = int(versionFile.read().strip())
+ except IOError, e:
+ log.error("Cannot open %s; skipping migration" %
+ (versionFilePath,))
+ except ValueError, e:
+ log.error("Invalid version number in %s; skipping migration" %
+ (versionFilePath,))
+
+ for upgradeVersion in range(onDiskVersion, newestVersion):
+ UpgradeTheServer.upgradeMethods[upgradeVersion](config)
+
+
@staticmethod
- def _doPrincipalCollectionInMemoryUpgrade():
+ def _doPrincipalCollectionInMemoryUpgrade(config):
# Look for the /principals/ directory on disk
old_principals = os.path.join(config.DocumentRoot, "principals")
if os.path.exists(old_principals):
# First move the proxy database and rename it
- UpgradeTheServer._doProxyDatabaseMoveUpgrade()
+ UpgradeTheServer._doProxyDatabaseMoveUpgrade(config)
# Now delete the on disk representation of principals
rmdir(old_principals)
@@ -46,7 +275,7 @@
)
@staticmethod
- def _doProxyDatabaseMoveUpgrade():
+ def _doProxyDatabaseMoveUpgrade(config):
# See if the old DB is present
old_db_path = os.path.join(config.DocumentRoot, "principals", CalendarUserProxyDatabase.dbOldFilename)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090219/989f8356/attachment-0001.html>
More information about the calendarserver-changes
mailing list