[CalendarServer-changes] [6221] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Aug 31 15:37:04 PDT 2010
Revision: 6221
http://trac.macosforge.org/projects/calendarserver/changeset/6221
Author: glyph at apple.com
Date: 2010-08-31 15:37:04 -0700 (Tue, 31 Aug 2010)
Log Message:
-----------
migrate calendars from the filesystem to the database automatically
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/txdav/caldav/datastore/util.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2010-08-31 22:35:31 UTC (rev 6220)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2010-08-31 22:37:04 UTC (rev 6221)
@@ -90,6 +90,7 @@
from txdav.common.datastore.sql import v1_schema
from txdav.base.datastore.subpostgres import PostgresService
+from txdav.caldav.datastore.util import UpgradeToDatabaseService
from twext.python.filepath import CachingFilePath
log = Logger()
@@ -355,8 +356,9 @@
Bind the UNIX socket and then change its group.
"""
super(GroupOwnedUNIXServer, self).privilegedStartService()
- fileName = self._port.port # Unfortunately, there's no public way to
- # access this. -glyph
+
+ # Unfortunately, there's no public way to access this. -glyph
+ fileName = self._port.port
os.chown(fileName, os.getuid(), self.gid)
@@ -376,6 +378,9 @@
def makeService(self, options):
+ """
+ Create the top-level service.
+ """
self.log_info("%s %s starting %s process..." % (self.description, version, config.ProcessType))
serviceMethod = getattr(self, "makeService_%s" % (config.ProcessType,), None)
@@ -402,7 +407,10 @@
# Process localization string files
processLocalizationFiles(config.Localization)
- # Now do any on disk upgrades we might need.
+ # Now do any on disk upgrades we might need. Note that this
+ # will only do the filesystem-format upgrades; migration to the
+ # database needs to be done when the connection and possibly
+ # server is already up and running. -glyph
upgradeData(config)
# Make sure proxies get initialized
@@ -414,8 +422,6 @@
return loader.updateProxyDB()
addSystemEventTrigger("after", "startup", _doProxyUpdate)
-
-
try:
service = serviceMethod(options)
except ConfigurationError, e:
@@ -480,6 +486,11 @@
def makeService_Slave(self, options):
+ """
+ Create a "slave" service, a subprocess of a service created with
+ L{makeService_Combined}, which does the work of actually handling
+ CalDAV and CardDAV requests.
+ """
#
# Change default log level to "info" as its useful to have
# that during startup
@@ -662,34 +673,75 @@
return service
+
def makeService_Single(self, options):
- #
- # Change default log level to "info" as its useful to have
- # that during startup
- #
+ """
+ Create a service to be used in a single-process, stand-alone
+ configuration.
+ """
+ return self.storageService(self.makeService_Slave(options))
- service = self.makeService_Slave(options)
- if config.UseDatabase:
- # Postgres
- dbRoot = CachingFilePath(config.DatabaseRoot)
+ def storageService(self, mainService, uid=None, gid=None):
+ """
+ If necessary, create a service to be started used for storage; for
+ example, starting a database backend. This service will then start the
+ main service.
- monitor = DelayedStartupProcessMonitor()
- service.processMonitor = monitor
+ This has the effect of delaying any child process spawning or
+ standalone port-binding until the backing for the selected data store
+ implementation is ready to process requests.
+ @param mainService: This is the service that will be doing the main
+ work of the current process. If the configured storage mode does
+ not require any particular setup, then this may return the
+ C{mainService} argument.
+
+ @type mainService: L{IService}
+
+ @param uid: the user ID to run the backend as, if this process is
+ running as root.
+ @type uid: C{int}
+
+ @param gid: the user ID to run the backend as, if this process is
+ running as root.
+ @type gid: C{int}
+
+ @return: the appropriate a service to start.
+
+ @rtype: L{IService}
+ """
+ if config.UseDatabase:
+ dbRoot = CachingFilePath(config.DatabaseRoot)
def subServiceFactory(connectionFactory):
- return monitor
+ # The database server is running at this point, so do the
+ # filesystem->database upgrade.
+ attachmentsRoot = dbRoot.child("attachments")
+ return UpgradeToDatabaseService.wrapService(
+ CachingFilePath(config.DocumentRoot), mainService,
+ connectionFactory, attachmentsRoot
+ )
+ if os.getuid() == 0: # Only override if root
+ postgresUID = uid
+ postgresGID = gid
+ else:
+ postgresUID = None
+ postgresGID = None
+ pgserv = PostgresService(
+ dbRoot, subServiceFactory, v1_schema, "caldav",
+ logFile=config.PostgresLogFile,
+ uid=postgresUID, gid=postgresGID
+ )
+ return pgserv
+ else:
+ return mainService
- postgresUID = None
- postgresGID = None
- PostgresService(dbRoot, subServiceFactory, v1_schema,
- "caldav", logFile=config.PostgresLogFile,
- uid=postgresUID, gid=postgresGID).setServiceParent(service)
-
- return service
-
def makeService_Combined(self, options):
+ """
+ Create a master service to coordinate a multi-process configuration,
+ spawning subprocesses that use L{makeService_Slave} to perform work.
+ """
s = ErrorLoggingMultiService()
# Make sure no old socket files are lying around.
@@ -705,7 +757,7 @@
if config.GroupName:
try:
gid = getgrnam(config.GroupName).gr_gid
- except KeyError, e:
+ except KeyError:
raise ConfigurationError("Invalid group name: %s" % (config.GroupName,))
else:
gid = os.getgid()
@@ -713,7 +765,7 @@
if config.UserName:
try:
uid = getpwnam(config.UserName).pw_uid
- except KeyError, e:
+ except KeyError:
raise ConfigurationError("Invalid user name: %s" % (config.UserName,))
else:
uid = os.getuid()
@@ -732,28 +784,8 @@
monitor = DelayedStartupProcessMonitor()
s.processMonitor = monitor
- if config.UseDatabase:
- # Postgres: delay spawning child processes until database is up
+ self.storageService(monitor, uid, gid).setServiceParent(s)
- dbRoot = CachingFilePath(config.DatabaseRoot)
-
- def subServiceFactory(connectionFactory):
- return monitor
-
- if os.getuid() == 0: # Only override if root
- postgresUID = uid
- postgresGID = gid
- else:
- postgresUID = None
- postgresGID = None
-
- PostgresService(dbRoot, subServiceFactory, v1_schema,
- "caldav", logFile=config.PostgresLogFile,
- uid=postgresUID, gid=postgresGID).setServiceParent(s)
-
- else:
- monitor.setServiceParent(s)
-
parentEnv = {
"PATH": os.environ.get("PATH", ""),
"PYTHONPATH": os.environ.get("PYTHONPATH", ""),
@@ -1138,7 +1170,6 @@
procmon.ProcessMonitor.__init__(self, *args, **kwargs)
self.processObjects = []
self._extraFDs = {}
- from twisted.internet import reactor
self.reactor = reactor
self.stopping = False
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2010-08-31 22:35:31 UTC (rev 6220)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2010-08-31 22:37:04 UTC (rev 6221)
@@ -78,6 +78,23 @@
+def storeFromConfig(config, notifierFactory=None):
+ """
+ Produce an L{IDataStore} from the given configuration and notifier factory.
+ """
+ if config.UseDatabase:
+ dbRoot = CachingFilePath(config.DatabaseRoot)
+ postgresService = PostgresService(dbRoot, None, v1_schema, "caldav",
+ logFile=config.PostgresLogFile)
+ return CommonSQLDataStore(postgresService.produceConnection,
+ notifierFactory, dbRoot.child("attachments"),
+ config.EnableCalDAV, config.EnableCardDAV)
+ else:
+ return CommonFileDataStore(FilePath(config.DocumentRoot),
+ notifierFactory, config.EnableCalDAV, config.EnableCardDAV)
+
+
+
def getRootResource(config, resources=None):
"""
Set up directory service and resource hierarchy based on config.
@@ -302,22 +319,14 @@
else:
notifierFactory = None
- if config.UseDatabase:
- _dbRoot = CachingFilePath(config.DatabaseRoot)
- _postgresService = PostgresService(_dbRoot, None, v1_schema, "caldav",
- logFile=config.PostgresLogFile)
- _newStore = CommonSQLDataStore(_postgresService.produceConnection,
- notifierFactory, _dbRoot.child("attachments"), config.EnableCalDAV, config.EnableCardDAV)
- else:
- _newStore = CommonFileDataStore(FilePath(config.DocumentRoot),
- notifierFactory, config.EnableCalDAV, config.EnableCardDAV)
+ newStore = storeFromConfig(config, notifierFactory)
if config.EnableCalDAV:
log.info("Setting up calendar collection: %r" % (calendarResourceClass,))
calendarCollection = calendarResourceClass(
directory,
"/calendars/",
- _newStore,
+ newStore,
)
if config.EnableCardDAV:
@@ -325,7 +334,7 @@
addressBookCollection = addressBookResourceClass(
directory,
"/addressbooks/",
- _newStore,
+ newStore,
)
directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
@@ -449,7 +458,7 @@
for path, cls, args, scheme in resources:
# putChild doesn't want "/" starting the path
- root.putChild(path, cls(root, _newStore, *args))
+ root.putChild(path, cls(root, newStore, *args))
# overrides requires "/" prepended
path = "/" + path
Modified: CalendarServer/trunk/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/util.py 2010-08-31 22:35:31 UTC (rev 6220)
+++ CalendarServer/trunk/txdav/caldav/datastore/util.py 2010-08-31 22:37:04 UTC (rev 6221)
@@ -16,6 +16,10 @@
"""
Utility logic common to multiple backend implementations.
"""
+from twext.python.log import LoggingMixIn
+from twisted.application.service import Service
+from txdav.common.datastore.file import CommonDataStore as FileStore
+from txdav.common.datastore.sql import CommonDataStore as SqlStore
from twext.python.vcomponent import InvalidICalendarDataError
from twext.python.vcomponent import VComponent
@@ -104,13 +108,16 @@
outCalendar.createCalendarObjectWithName(
calendarObject.name(),
calendarObject.component()) # XXX WRONG SHOULD CALL getComponent
+
+ # Only the owner's properties are migrated, since previous releases of
+ # calendar server didn't have per-user properties.
outCalendar.calendarObjectWithName(
calendarObject.name()).properties().update(
calendarObject.properties())
# XXX attachments
-def migrateHome(inHome, outHome, getComponent):
+def migrateHome(inHome, outHome, getComponent=lambda x: x.component()):
"""
Copy all calendars and properties in the given input calendar to the given
output calendar.
@@ -133,3 +140,85 @@
outHome.createCalendarWithName(name)
outCalendar = outHome.calendarWithName(name)
_migrateCalendar(calendar, outCalendar, getComponent)
+ # No migration for notifications, since they weren't present in earlier
+ # released versions of CalendarServer.
+
+
+# TODO: implement addressbooks, import from txdav.common.datastore.file
+TOPPATHS = ['calendars']
+
+class UpgradeToDatabaseService(Service, LoggingMixIn, object):
+ """
+ Upgrade resources from a filesystem store to a database store.
+ """
+
+
+ @classmethod
+ def wrapService(cls, path, service, connectionFactory, sqlAttachmentsPath):
+ """
+ Create an L{UpgradeToDatabaseService} if there are still file-based
+ calendar or addressbook homes remaining in the given path.
+
+ Maintenance note: we may want to pass a SQL store in directly rather
+ than the combination of connection factory and attachments path, since
+ there always must be a SQL store, but the path should remain a path
+ because there may not I{be} a file-backed store present and we should
+ not create it as a result of checking for it.
+
+ @param path: a path pointing at the document root.
+ @type path: L{CachingFilePath}
+
+ @param service: the service to wrap. This service should be started
+ when the upgrade is complete. (This is accomplished by returning
+ it directly when no upgrade needs to be done, and by adding it to
+ the service hierarchy when the upgrade completes; assuming that the
+ service parent of the resulting service will be set to a
+ L{MultiService} or similar.)
+
+ @type service: L{IService}
+
+ @return: a service
+ @rtype: L{IService}
+ """
+ for homeType in TOPPATHS:
+ if path.child(homeType).exists():
+ self = cls(
+ FileStore(path, None, True, True),
+ SqlStore(connectionFactory, None, sqlAttachmentsPath,
+ True, True),
+ service
+ )
+ return self
+ return service
+
+
+ def __init__(self, fileStore, sqlStore, service):
+ """
+ Initialize the service.
+ """
+ self.wrappedService = service
+ self.fileStore = fileStore
+ self.sqlStore = sqlStore
+
+
+ def startService(self):
+ self.log_warn("Beginning filesystem -> database upgrade.")
+ for fileTxn, fileHome in self.fileStore.eachCalendarHome():
+ uid = fileHome.uid()
+ self.log_warn("Migrating UID %r" % (uid,))
+ sqlTxn = self.sqlStore.newTransaction()
+ sqlHome = sqlTxn.calendarHomeWithUID(uid, create=True)
+ migrateHome(fileHome, sqlHome)
+ fileTxn.commit()
+ sqlTxn.commit()
+ # FIXME: need a public remove...HomeWithUID() for de-provisioning
+ fileHome._path.remove()
+ for homeType in TOPPATHS:
+ homesPath = self.fileStore._path.child(homeType)
+ if homesPath.isdir():
+ homesPath.remove()
+ self.log_warn(
+ "Filesystem upgrade complete, launching database service."
+ )
+ self.wrappedService.setServiceParent(self.parent)
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100831/6f44f9b5/attachment-0001.html>
More information about the calendarserver-changes
mailing list