[CalendarServer-changes] [6435] CalendarServer/trunk/contrib/migration/59_calendarmigrator.py
source_changes at macosforge.org
source_changes at macosforge.org
Mon Oct 18 09:32:54 PDT 2010
Revision: 6435
http://trac.macosforge.org/projects/calendarserver/changeset/6435
Author: sagen at apple.com
Date: 2010-10-18 09:32:53 -0700 (Mon, 18 Oct 2010)
Log Message:
-----------
First pass at migrating/merging AB and iCal config/data.
Modified Paths:
--------------
CalendarServer/trunk/contrib/migration/59_calendarmigrator.py
Modified: CalendarServer/trunk/contrib/migration/59_calendarmigrator.py
===================================================================
--- CalendarServer/trunk/contrib/migration/59_calendarmigrator.py 2010-10-18 04:35:00 UTC (rev 6434)
+++ CalendarServer/trunk/contrib/migration/59_calendarmigrator.py 2010-10-18 16:32:53 UTC (rev 6435)
@@ -21,8 +21,10 @@
from __future__ import with_statement
import datetime
+import grp
import optparse
import os
+import pwd
import shutil
import sys
@@ -34,7 +36,134 @@
LAUNCHD_OVERRIDES = "var/db/launchd.db/com.apple.launchd/overrides.plist"
LAUNCHD_PREFS_DIR = "System/Library/LaunchDaemons"
CALDAVD_CONFIG_DIR = "private/etc/caldavd"
+CARDDAVD_CONFIG_DIR = "private/etc/carddavd"
+CALDAVD_PLIST = "caldavd.plist"
+CARDDAVD_PLIST = "carddavd.plist"
+NEW_SERVER_ROOT = "/Library/Server/Calendar and Contacts"
+
+verbatimKeys = """
+AccountingCategories
+AccountingPrincipals
+AdminPrincipals
+Aliases
+AnonymousDirectoryAddressBookAccess
+AugmentService
+BindAddresses
+ConfigRoot
+ControlPort
+DatabaseRoot
+DefaultLogLevel
+DirectoryAddressBook
+DirectoryService
+EnableAddMember
+EnableAnonymousReadNav
+EnableCalDAV
+EnableCardDAV
+EnableDropBox
+EnableExtendedAccessLog
+EnableKeepAlive
+EnableMonolithicCalendars
+EnablePrincipalListings
+EnablePrivateEvents
+EnableProxyPrincipals
+EnableSACLs
+EnableSSL
+EnableSearchAddressBook
+EnableSyncReport
+EnableTimezoneService
+EnableWebAdmin
+EnableWellKnown
+ErrorLogEnabled
+ErrorLogMaxRotatedFiles
+ErrorLogRotateMB
+FreeBusyURL
+GlobalAddressBook
+GlobalStatsLoggingFrequency
+GlobalStatsLoggingPeriod
+GlobalStatsSocket
+GroupName
+HTTPPort
+HTTPRetryAfter
+IdleConnectionTimeOut
+Includes
+ListenBacklog
+Localization
+LogLevels
+LogRoot
+MaxAccepts
+MaxAttendeesPerInstance
+MaxInstancesForRRULE
+MaxMultigetWithDataHREFs
+MaxQueryWithDataResults
+MaxRequests
+MaximumAttachmentSize
+Memcached
+MultiProcess
+Notifications
+Partitioning
+Postgres
+ProcessType
+Profiling
+ProxyDBService
+ProxyLoadFromFile
+ReadPrincipals
+RedirectHTTPToHTTPS
+RejectClients
+ResourceService
+ResponseCompression
+RotateAccessLog
+RunRoot
+SSLAuthorityChain
+SSLCertAdmin
+SSLCertificate
+SSLCiphers
+SSLMethod
+SSLPrivateKey
+Scheduling
+ServerHostName
+ServerRoot
+Sharing
+SudoersFile
+Twisted
+UIDReservationTimeOut
+UseDatabase
+UseMetaFD
+UserName
+UserQuota
+WebCalendarRoot
+umask
+""".split()
+
+# These are going to require some processing
+specialKeys = """
+AccessLogFile
+AccountingLogRoot
+Authentication
+BindHTTPPorts
+BindSSLPorts
+DataRoot
+DocumentRoot
+ErrorLogFile
+MaxAddressBookMultigetHrefs
+MaxAddressBookQueryResults
+""".split()
+
+ignoredKkeys = """
+ControlSocket
+EnableAnonymousReadRoot
+EnableFindSharedReport
+EnableNotifications
+PIDFile
+PythonDirector
+ResponseCacheTimeout
+SSLPassPhraseDialog
+SSLPort
+ServerStatsFile
+Verbose
+""".split()
+
+
def main():
optionParser = optparse.OptionParser()
@@ -65,11 +194,13 @@
help='language identifier (IGNORED)')
(options, args) = optionParser.parse_args()
+ log("Options: %s" % (options,))
if options.sourceRoot:
if os.path.exists(options.sourceRoot):
- migrateConfiguration(options)
+ newServerRootValue = migrateData(options)
+ migrateConfiguration(options, newServerRootValue)
migrateRunState(options)
else:
@@ -95,106 +226,111 @@
setServiceStateDisabled(options.targetRoot, LAUNCHD_KEY, disabled)
-def migrateConfiguration(options):
+def migrateConfiguration(options, newServerRootValue):
"""
Copy files/directories/symlinks from previous system's /etc/caldavd
+ and /etc/carddavd
Skips anything ending in ".default".
Regular files overwrite copies in new system.
Directories and symlinks only copied over if they don't overwrite anything.
"""
- oldConfigDir = os.path.join(options.sourceRoot, CALDAVD_CONFIG_DIR)
- if not os.path.exists(oldConfigDir):
- log("Old configuration directory does not exist: %s" % (oldConfigDir,))
- return
-
newConfigDir = os.path.join(options.targetRoot, CALDAVD_CONFIG_DIR)
if not os.path.exists(newConfigDir):
log("New configuration directory does not exist: %s" % (newConfigDir,))
return
- log("Copying configuration files from %s to %s" % (oldConfigDir, newConfigDir))
- for name in os.listdir(oldConfigDir):
+ for configDir in (CALDAVD_CONFIG_DIR, CARDDAVD_CONFIG_DIR):
- if not name.endswith(".default"):
+ oldConfigDir = os.path.join(options.sourceRoot, configDir)
+ if not os.path.exists(oldConfigDir):
+ log("Old configuration directory does not exist: %s" % (oldConfigDir,))
+ continue
- oldPath = os.path.join(oldConfigDir, name)
- newPath = os.path.join(newConfigDir, name)
- if os.path.islink(oldPath) and not os.path.exists(newPath):
- # Recreate the symlink if it won't overwrite an existing file
- link = os.readlink(oldPath)
- log("Symlinking %s to %s" % (newPath, link))
- os.symlink(link, newPath)
+ log("Copying configuration files from %s to %s" % (oldConfigDir, newConfigDir))
- elif os.path.isfile(oldPath):
+ for name in os.listdir(oldConfigDir):
- if name == "caldavd.plist":
- # Migrate certain settings from the old plist to new:
- log("Parsing %s" % (oldPath,))
- oldPlist = readPlist(oldPath)
- if os.path.exists(newPath):
- log("Parsing %s" % (newPath,))
- newPlist = readPlist(newPath)
- log("Removing %s" % (newPath,))
- os.remove(newPath)
- else:
- newPlist = { }
- log("Processing %s" % (oldPath,))
- mergePlist(oldPlist, newPlist)
- log("Writing %s" % (newPath,))
- writePlist(newPlist, newPath)
+ if not (name.endswith(".default") or name in (CALDAVD_PLIST, CARDDAVD_PLIST)):
- else:
+ oldPath = os.path.join(oldConfigDir, name)
+ newPath = os.path.join(newConfigDir, name)
+
+ if os.path.islink(oldPath) and not os.path.exists(newPath):
+ # Recreate the symlink if it won't overwrite an existing file
+ link = os.readlink(oldPath)
+ log("Symlinking %s to %s" % (newPath, link))
+ os.symlink(link, newPath)
+
+ elif os.path.isfile(oldPath):
# Copy the file over, overwriting copy in newConfigDir
log("Copying file %s to %s" % (oldPath, newConfigDir))
shutil.copy2(oldPath, newConfigDir)
+ elif os.path.isdir(oldPath) and not os.path.exists(newPath):
+ # Copy the dir over, but only if new one doesn't exist
+ log("Copying directory %s to %s" % (oldPath, newPath))
+ shutil.copytree(oldPath, newPath, symlinks=True)
- elif os.path.isdir(oldPath) and not os.path.exists(newPath):
- # Copy the dir over, but only if new one doesn't exist
- log("Copying directory %s to %s" % (oldPath, newPath))
- shutil.copytree(oldPath, newPath, symlinks=True)
-def mergePlist(oldPlist, newPlist):
+ # Migrate certain settings from the old plists to new:
- # The following CalendarServer v1.x keys are ignored:
- # EnableNotifications, Verbose
+ oldCalDAVPlistPath = os.path.join(options.sourceRoot, CALDAVD_CONFIG_DIR,
+ CALDAVD_PLIST)
+ if os.path.exists(oldCalDAVPlistPath):
+ oldCalDAVDPlist = readPlist(oldCalDAVPlistPath)
+ else:
+ oldCalDAVDPlist = { }
+ oldCardDAVDPlistPath = os.path.join(options.sourceRoot, CARDDAVD_CONFIG_DIR,
+ CARDDAVD_PLIST)
+ if os.path.exists(oldCardDAVDPlistPath):
+ oldCardDAVDPlist = readPlist(oldCardDAVDPlistPath)
+ else:
+ oldCardDAVDPlist = { }
+
+ newCalDAVDPlistPath = os.path.join(options.targetRoot, CALDAVD_CONFIG_DIR,
+ CALDAVD_PLIST)
+ if os.path.exists(newCalDAVDPlistPath):
+ newCalDAVDPlist = readPlist(newCalDAVDPlistPath)
+ else:
+ newCalDAVDPlist = { }
+
+ log("Processing %s and %s" % (oldCalDAVPlistPath, oldCardDAVDPlistPath))
+ mergePlist(oldCalDAVDPlist, oldCardDAVDPlist, newCalDAVDPlist)
+
+ newCalDAVDPlist["ServerRoot"] = newServerRootValue
+ newCalDAVDPlist["DocumentRoot"] = "Documents"
+ newCalDAVDPlist["DataRoot"] = "Data"
+
+ log("Writing %s" % (newCalDAVDPlistPath,))
+ writePlist(newCalDAVDPlist, newCalDAVDPlistPath)
+
+
+def mergePlist(oldCalDAVDPlist, oldCardDAVDPlist, newCalDAVDPlist):
+
# These keys are copied verbatim:
- for key in (
- "AccessLogFile", "AdminPrincipals", "BindAddresses", "BindHTTPPorts",
- "BindSSLPorts", "ControlSocket", "DocumentRoot", "EnableDropBox",
- "EnableProxyPrincipals", "EnableSACLs", "ErrorLogFile", "GroupName",
- "HTTPPort", "MaximumAttachmentSize", "MultiProcess", "PIDFile",
- "ProcessType", "ResponseCompression", "RotateAccessLog",
- "SSLAuthorityChain", "SSLCertificate", "SSLPort", "SSLPrivateKey",
- "ServerHostName", "ServerStatsFile", "SudoersFile", "UserName",
- "UserQuota",
- ):
- if key in oldPlist:
- newPlist[key] = oldPlist[key]
+ for key in verbatimKeys:
+ if key in oldCardDAVDPlist:
+ newCalDAVDPlist[key] = oldCardDAVDPlist[key]
+ if key in oldCalDAVDPlist:
+ newCalDAVDPlist[key] = oldCalDAVDPlist[key]
# "Wiki" is a new authentication in v2.x; copy all "Authentication" sub-keys # over, and "Wiki" will be picked up from the new plist:
- if "Authentication" in oldPlist:
- for key in oldPlist["Authentication"]:
- newPlist["Authentication"][key] = oldPlist["Authentication"][key]
+ if "Authentication" in oldCalDAVDPlist:
+ for key in oldCalDAVDPlist["Authentication"]:
+ newCalDAVDPlist["Authentication"][key] = oldCalDAVDPlist["Authentication"][key]
# Strip out any unknown params from the DirectoryService:
- if "DirectoryService" in oldPlist:
- newPlist["DirectoryService"] = oldPlist["DirectoryService"]
- for key in newPlist["DirectoryService"]["params"].keys():
- if key not in (
- "node",
- "cacheTimeout", "xmlFile"
- ):
- del newPlist["DirectoryService"]["params"][key]
+ if "DirectoryService" in oldCalDAVDPlist:
+ newCalDAVDPlist["DirectoryService"] = oldCalDAVDPlist["DirectoryService"]
+ for key in newCalDAVDPlist["DirectoryService"]["params"].keys():
+ if key not in ("node", "cacheTimeout", "xmlFile"):
+ del newCalDAVDPlist["DirectoryService"]["params"][key]
- # Place DataRoot as a sibling of DocumentRoot:
- parent = os.path.dirname(newPlist["DocumentRoot"].rstrip("/"))
- newPlist["DataRoot"] = os.path.join(parent, "Data")
def isServiceDisabled(source, service):
@@ -260,6 +396,259 @@
# Could not write to log
pass
+def migrateData(options):
+ """
+ Examines the old caldavd.plist and carddavd.plist to see where data
+ lives in the previous system. If there is old data, calls relocateData( )
+ """
+ oldCalDocuments = None
+ oldCalData = None
+ oldABDocuments = None
+ calendarDataInDefaultLocation = True
+ addressbookDataInDefaultLocation = True
+ uid = -1
+ gid = -1
+ newServerRoot = None
+
+ oldCalConfigDir = os.path.join(options.sourceRoot, CALDAVD_CONFIG_DIR)
+ oldCalPlistPath = os.path.join(oldCalConfigDir, CALDAVD_PLIST)
+ if os.path.exists(oldCalPlistPath):
+ oldCalPlist = readPlist(oldCalPlistPath)
+ uid, gid = getServerIDs(oldCalPlist)
+ log("ServerIDs: %d, %d" % (uid, gid))
+ else:
+ log("Can't find previous calendar plist at %s" % (oldCalPlistPath,))
+ oldCalPlist = None
+ newCalConfigDir = os.path.join(options.targetRoot, CALDAVD_CONFIG_DIR)
+ newCalPlistPath = os.path.join(newCalConfigDir, CALDAVD_PLIST)
+ if os.path.exists(newCalPlistPath):
+ newCalPlist = readPlist(newCalPlistPath)
+ uid, gid = getServerIDs(newCalPlist)
+ log("ServerIDs: %d, %d" % (uid, gid))
+
+
+ oldABConfigDir = os.path.join(options.sourceRoot, CARDDAVD_CONFIG_DIR)
+ oldABPlistPath = os.path.join(oldABConfigDir, CARDDAVD_PLIST)
+ if os.path.exists(oldABPlistPath):
+ oldABPlist = readPlist(oldABPlistPath)
+ else:
+ log("Can't find previous addressbook plist at %s" % (oldABPlistPath,))
+ oldABPlist = None
+
+ if oldCalPlist is not None:
+ # See if there is actually any calendar data
+
+ oldDocumentRoot = oldCalPlist["DocumentRoot"]
+ if oldDocumentRoot.rstrip("/") != "/Library/CalendarServer/Documents":
+ log("Calendar data in non-standard location: %s" % (oldDocumentRoot,))
+ calendarDataInDefaultLocation = False
+ else:
+ log("Calendar data in standard location: %s" % (oldDocumentRoot,))
+
+ oldDataRoot = oldCalPlist["DataRoot"]
+
+ oldCalendarsPath = os.path.join(oldDocumentRoot, "calendars")
+ if os.path.exists(oldCalendarsPath):
+ # There is calendar data
+ oldCalDocuments = oldDocumentRoot
+ oldCalData = oldDataRoot
+ log("Calendar data to migrate from %s and %s" %
+ (oldCalDocuments, oldCalData))
+
+ if calendarDataInDefaultLocation:
+ newServerRoot = absolutePathWithRoot(options.targetRoot,
+ NEW_SERVER_ROOT)
+ newServerRootValue = NEW_SERVER_ROOT
+ else:
+ newServerRoot = absolutePathWithRoot(options.targetRoot,
+ oldDocumentRoot)
+ newServerRootValue = oldDocumentRoot
+ else:
+ log("No calendar data to migrate")
+
+ if oldABPlist is not None:
+ # See if there is actually any addressbook data
+
+ oldDocumentRoot = oldABPlist["DocumentRoot"]
+ if oldDocumentRoot.rstrip("/") != "/Library/AddressBookServer/Documents":
+ log("AddressBook data in non-standard location: %s" % (oldDocumentRoot,))
+ addressbookDataInDefaultLocation = False
+ else:
+ log("AddressBook data in standard location: %s" % (oldDocumentRoot,))
+
+ oldAddressbooksPath = os.path.join(oldDocumentRoot, "addressbooks")
+ if os.path.exists(oldAddressbooksPath):
+ # There is addressbook data
+ oldABDocuments = oldDocumentRoot
+ log("AddressBook data to migrate from %s" % (oldABDocuments,))
+
+ if newServerRoot is None:
+ # don't override server root computed from calendar
+ if addressbookDataInDefaultLocation:
+ newServerRoot = absolutePathWithRoot(options.targetRoot,
+ NEW_SERVER_ROOT)
+ newServerRootValue = NEW_SERVER_ROOT
+ else:
+ newServerRoot = absolutePathWithRoot(options.targetRoot,
+ oldDocumentRoot)
+ newServerRootValue = oldDocumentRoot
+ else:
+ log("No addressbook data to migrate")
+
+ if (oldCalDocuments or oldABDocuments) and newServerRoot:
+ relocateData(oldCalDocuments, oldCalData, oldABDocuments, uid, gid,
+ calendarDataInDefaultLocation, addressbookDataInDefaultLocation,
+ newServerRoot)
+
+ return newServerRootValue
+
+def relocateData(oldCalDocuments, oldCalData, oldABDocuments, uid, gid,
+ calendarDataInDefaultLocation, addressbookDataInDefaultLocation,
+ newServerRoot):
+ """
+ Relocates existing calendar data to the new default location iff the data
+ was previously in the old default location; otherwise the old calendar
+ DocumentRoot becomes the new ServerRoot directory, the contents of the
+ old DocumentRoot are moved into ServerRoot/Documents and the contents of
+ old DataRoot are copied/moved into ServerRoot/Data. If there is addressbook
+ data, a symlink is created as ServerRoot/Documents/addressbooks pointing
+ to the old addressbook directory so that the import-to-PostgreSQL will
+ find it.
+ """
+
+ log("RelocateData: cal documents=%s, cal data=%s, ab documents=%s, new server root=%s"
+ % (oldCalDocuments, oldCalData, oldABDocuments, newServerRoot))
+
+ if oldCalDocuments and os.path.exists(oldCalDocuments):
+
+ if calendarDataInDefaultLocation:
+ # We're in the default location, relocate to new location
+ newCalDocuments = os.path.join(newServerRoot, "Documents")
+ newCalData = os.path.join(newServerRoot, "Data")
+ if os.path.exists(oldCalDocuments):
+ if os.path.exists(newCalDocuments):
+ # Move evertying from oldCalDocuments
+ for item in list(os.listdir(oldCalDocuments)):
+ source = os.path.join(oldCalDocuments, item)
+ dest = os.path.join(newCalDocuments, item)
+ log("Relocating %s to %s" % (source, dest))
+ os.rename(source, dest)
+ else:
+ log("Error: %s does not exist" % (newCalDocuments,))
+ else:
+ log("Warning: %s does not exist; nothing to migrate" % (oldCalDocuments,))
+ else:
+ # The admin has moved calendar data to a non-standard location so
+ # we're going to leave it there, but move things down a level so
+ # that the old DocumentRoot becomes new ServerRoot
+
+ # Create "Documents" directory with same ownership as oldCalDocuments
+ newCalDocuments = os.path.join(newServerRoot, "Documents")
+ log("New documents directory: %s" % (newCalDocuments,))
+ newCalData = os.path.join(newServerRoot, "Data")
+ log("New data directory: %s" % (newCalData,))
+ os.mkdir(newCalDocuments)
+ os.mkdir(newCalData)
+ for item in list(os.listdir(newServerRoot)):
+ if item not in ("Documents", "Data"):
+ source = os.path.join(newServerRoot, item)
+ dest = os.path.join(newCalDocuments, item)
+ log("Relocating %s to %s" % (source, dest))
+ os.rename(source, dest)
+
+ # Relocate calendar DataRoot, copying all files
+ if os.path.exists(oldCalData):
+ for item in list(os.listdir(oldCalData)):
+ source = os.path.join(oldCalData, item)
+ if not os.path.isfile(source):
+ continue
+ dest = os.path.join(newCalData, item)
+ log("Relocating %s to %s" % (source, dest))
+ try:
+ os.rename(source, dest)
+ except OSError:
+ # Can't rename because it's cross-volume; must copy/delete
+ shutil.copy2(source, dest)
+ os.remove(source)
+
+ # Symlink to AB document root so server will find it an import to
+ # PostgreSQL
+ if oldABDocuments and os.path.exists(oldABDocuments):
+ oldAddressBooks = os.path.join(oldABDocuments, "addressbooks")
+ newAddressBooks = os.path.join(newCalDocuments, "addressbooks")
+ log("Symlinking AddressBook data: %s to %s" % (newAddressBooks, oldAddressBooks))
+ os.symlink(oldAddressBooks, newAddressBooks)
+
+
+ elif oldABDocuments and os.path.exists(oldABDocuments):
+ # No calendar data, only addressbook data
+
+ if addressbookDataInDefaultLocation:
+ # We're in the default location, relocate to new location
+ newABDocuments = os.path.join(newServerRoot, "Documents")
+ if os.path.exists(newABDocuments):
+ # Move evertying from oldABDocuments
+ for item in list(os.listdir(oldABDocuments)):
+ source = os.path.join(oldABDocuments, item)
+ dest = os.path.join(newABDocuments, item)
+ log("Relocating %s to %s" % (source, dest))
+ os.rename(source, dest)
+ else:
+ log("Error: %s does not exist" % (newABDocuments,))
+ else:
+ # The admin has moved addressbook data to a non-standard location so
+ # we're going to leave it there, but move things down a level so
+ # that the old DocumentRoot becomes new ServerRoot
+
+ # Create "Documents" directory with same ownership as oldABDocuments
+ newABDocuments = os.path.join(newServerRoot, "Documents")
+ newABData = os.path.join(newServerRoot, "Data")
+ log("New documents directory: %s" % (newABDocuments,))
+ os.mkdir(newABDocuments)
+ os.mkdir(newABData)
+ for item in list(os.listdir(newServerRoot)):
+ if item not in ("Documents", "Data"):
+ source = os.path.join(newServerRoot, item)
+ dest = os.path.join(newABDocuments, item)
+ log("Relocating %s to %s" % (source, dest))
+ os.rename(source, dest)
+
+ if newServerRoot and os.path.exists(newServerRoot):
+ """
+ Change onwnership of entire ServerRoot
+ """
+ os.chown(newServerRoot, uid, gid)
+ for root, dirs, files in os.walk(newServerRoot, followlinks=True):
+ for name in dirs:
+ os.chown(os.path.join(root, name), uid, gid)
+ for name in files:
+ os.chown(os.path.join(root, name), uid, gid)
+
+
+
+def getServerIDs(plist):
+ """
+ Given a caldavd.plist, return the userid and groupid for the UserName and
+ GroupName specified.
+ """
+ uid = -1
+ if plist["UserName"]:
+ uid = pwd.getpwnam(plist["UserName"]).pw_uid
+ gid = -1
+ if plist["GroupName"]:
+ gid = grp.getgrnam(plist["GroupName"]).gr_gid
+ return uid, gid
+
+def absolutePathWithRoot(root, path):
+ """
+ Combine root and path as long as path does not start with /Volumes/
+ """
+ if path.startswith("/Volumes/"):
+ return path
+ else:
+ path = path.strip("/")
+ return os.path.join(root, path)
+
if __name__ == '__main__':
main()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20101018/8cac933a/attachment-0001.html>
More information about the calendarserver-changes
mailing list