[CalendarServer-changes] [10762] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Feb 19 14:15:27 PST 2013
Revision: 10762
http://trac.calendarserver.org//changeset/10762
Author: cdaboo at apple.com
Date: 2013-02-19 14:15:27 -0800 (Tue, 19 Feb 2013)
Log Message:
-----------
Set a default option to disallow server startup if a DB upgrade is needed. Modify the cs_upgrade tool to override
that to always do an upgrade, and also add a --status option to check the DB status (without forcing an upgrade).
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tools/cmdline.py
CalendarServer/trunk/calendarserver/tools/upgrade.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2013-02-19 22:13:57 UTC (rev 10761)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2013-02-19 22:15:27 UTC (rev 10762)
@@ -539,7 +539,6 @@
)
self.monitor.addProcessObject(process, PARENT_ENVIRONMENT)
-
if config.GroupCaching.Enabled and config.GroupCaching.EnableUpdater:
self.maker.log_info("Adding group caching service")
@@ -715,7 +714,6 @@
if pool is not None:
pool.setServiceParent(result)
-
# Optionally set up push notifications
if config.Notifications.Enabled:
pushService = PushService.makeService(config.Notifications, store)
@@ -1111,6 +1109,7 @@
store, uid=overrideUID, gid=overrideGID,
),
store, uid=overrideUID, gid=overrideGID,
+ failIfUpgradeNeeded=config.FailIfUpgradeNeeded,
)
)
upgradeSvc.setServiceParent(ms)
Modified: CalendarServer/trunk/calendarserver/tools/cmdline.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/cmdline.py 2013-02-19 22:13:57 UTC (rev 10761)
+++ CalendarServer/trunk/calendarserver/tools/cmdline.py 2013-02-19 22:15:27 UTC (rev 10762)
@@ -30,7 +30,7 @@
# TODO: direct unit tests for these functions.
-def utilityMain(configFileName, serviceClass, reactor=None, serviceMaker=CalDAVServiceMaker, verbose=False):
+def utilityMain(configFileName, serviceClass, reactor=None, serviceMaker=CalDAVServiceMaker, patchConfig=None, onShutdown=None, verbose=False):
"""
Shared main-point for utilities.
@@ -52,6 +52,11 @@
provides L{ICalendarStore} and/or L{IAddressbookStore} and returns an
L{IService}.
+ @param patchConfig: a 1-argument callable which takes a config object
+ and makes and changes necessary for the tool.
+
+ @param onShutdown: a 0-argument callable which will run on shutdown.
+
@param reactor: if specified, the L{IReactorTime} / L{IReactorThreads} /
L{IReactorTCP} (etc) provider to use. If C{None}, the default reactor
will be imported and used.
@@ -66,6 +71,8 @@
from twisted.internet import reactor
try:
config = loadConfig(configFileName)
+ if patchConfig is not None:
+ patchConfig(config)
# If we don't have permission to access the DataRoot directory, we
# can't proceed. If this fails it should raise OSError which we
@@ -83,6 +90,8 @@
reactor.addSystemEventTrigger("during", "startup", service.startService)
reactor.addSystemEventTrigger("before", "shutdown", service.stopService)
+ if onShutdown is not None:
+ reactor.addSystemEventTrigger("before", "shutdown", onShutdown)
except (ConfigurationError, OSError), e:
sys.stderr.write("Error: %s\n" % (e,))
Modified: CalendarServer/trunk/calendarserver/tools/upgrade.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/upgrade.py 2013-02-19 22:13:57 UTC (rev 10761)
+++ CalendarServer/trunk/calendarserver/tools/upgrade.py 2013-02-19 22:15:27 UTC (rev 10762)
@@ -63,7 +63,7 @@
"""
Command-line options for 'calendarserver_upgrade'
- @ivar upgradeers: a list of L{DirectoryUpgradeer} objects which can identify the
+ @ivar upgraders: a list of L{DirectoryUpgradeer} objects which can identify the
calendars to upgrade, given a directory service. This list is built by
parsing --record and --collection options.
"""
@@ -71,6 +71,7 @@
synopsis = description
optFlags = [
+ ['status', 's', "Check database status and exit."],
['postprocess', 'p', "Perform post-database-import processing."],
['debug', 'D', "Debug logging."],
]
@@ -121,6 +122,8 @@
Service which runs, exports the appropriate records, then stops the reactor.
"""
+ started = False
+
def __init__(self, store, options, output, reactor, config):
super(UpgraderService, self).__init__()
self.store = store
@@ -135,7 +138,13 @@
"""
Immediately stop. The upgrade will have been run before this.
"""
- self.output.write("Upgrade complete, shutting down.\n")
+ # If we get this far the database is OK
+ if self.options["status"]:
+ self.output.write("Database OK.\n")
+ else:
+ self.output.write("Upgrade complete, shutting down.\n")
+ UpgraderService.started = True
+
from twisted.internet import reactor
from twisted.internet.error import ReactorNotRunning
try:
@@ -188,16 +197,29 @@
output.write(logDateString() + ' ' + log.textFromEventDict(event) + "\n")
output.flush()
- setLogLevelForNamespace(None, "debug")
- log.addObserver(onlyUpgradeEvents)
+ if not options["status"]:
+ setLogLevelForNamespace(None, "debug")
+ log.addObserver(onlyUpgradeEvents)
+
+
def customServiceMaker():
customService = CalDAVServiceMaker()
customService.doPostImport = options["postprocess"]
return customService
- utilityMain(options["config"], makeService, reactor, customServiceMaker, verbose=options["debug"])
+ def _patchConfig(config):
+ config.FailIfUpgradeNeeded = options["status"]
+
+ def _onShutdown():
+ if not UpgraderService.started:
+ print "Failed to start service."
+
+ utilityMain(options["config"], makeService, reactor, customServiceMaker, patchConfig=_patchConfig, onShutdown=_onShutdown, verbose=options["debug"])
+
+
+
def logDateString():
logtime = time.localtime()
Y, M, D, h, m, s = logtime[:6]
@@ -219,3 +241,6 @@
return '-%02d%02d' % (h, m)
else:
return '+%02d%02d' % (h, m)
+
+if __name__ == '__main__':
+ main()
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-02-19 22:13:57 UTC (rev 10761)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-02-19 22:15:27 UTC (rev 10762)
@@ -302,6 +302,10 @@
# the master process, rather than having
# each client make its connections directly.
+ "FailIfUpgradeNeeded" : True, # Set to True to prevent the server or utility tools
+ # tools from running if the database needs a schema
+ # upgrade.
+
#
# Types of service provided
#
@@ -1342,14 +1346,14 @@
if reloading:
return
- for key, service in configDict.Notifications["Services"].iteritems():
+ for _ignore_key, service in configDict.Notifications["Services"].iteritems():
if service["Enabled"]:
configDict.Notifications["Enabled"] = True
break
else:
configDict.Notifications["Enabled"] = False
- for key, service in configDict.Notifications["Services"].iteritems():
+ for _ignore_key, service in configDict.Notifications["Services"].iteritems():
if (
service["Service"] == "calendarserver.push.applepush.ApplePushNotifierService" and
@@ -1386,7 +1390,7 @@
except KeychainPasswordNotFound:
# The password doesn't exist in the keychain.
log.info("%s APN certificate passphrase not found in keychain" % (protocol,))
-
+
if (
service["Service"] == "calendarserver.push.amppush.AMPPushNotifierService" and
service["Enabled"]
@@ -1397,7 +1401,6 @@
-
def _updateScheduling(configDict, reloading=False):
#
# Scheduling
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2013-02-19 22:13:57 UTC (rev 10761)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2013-02-19 22:15:27 UTC (rev 10762)
@@ -23,7 +23,7 @@
from twisted.python.modules import getModule
from twisted.trial.unittest import TestCase
from txdav.common.datastore.test.util import theStoreBuilder, StubNotifierFactory
-from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService,\
+from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService, \
UpgradeDatabaseDataService
import re
@@ -42,19 +42,20 @@
return 1
return int(found.group(2))
+
def test_scanUpgradeFiles(self):
-
+
upgrader = UpgradeDatabaseSchemaService(None, None)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
files = upgrader.scanForUpgradeFiles("fake_dialect")
- self.assertEqual(files,
+ self.assertEqual(files,
[(3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"))],
)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
files = upgrader.scanForUpgradeFiles("fake_dialect")
- self.assertEqual(files,
+ self.assertEqual(files,
[
(3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")),
(3, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")),
@@ -62,14 +63,15 @@
]
)
+
def test_determineUpgradeSequence(self):
-
+
upgrader = UpgradeDatabaseSchemaService(None, None)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 4, files, "fake_dialect")
- self.assertEqual(upgrades,
+ self.assertEqual(upgrades,
[upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")],
)
self.assertRaises(RuntimeError, upgrader.determineUpgradeSequence, 3, 5, files, "fake_dialect")
@@ -77,35 +79,36 @@
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
- self.assertEqual(upgrades,
+ self.assertEqual(upgrades,
[upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")]
)
upgrades = upgrader.determineUpgradeSequence(4, 5, files, "fake_dialect")
- self.assertEqual(upgrades,
+ self.assertEqual(upgrades,
[upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")]
)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema3")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
- self.assertEqual(upgrades,
+ self.assertEqual(upgrades,
[
upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"),
upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql"),
]
)
+
def test_upgradeAvailability(self):
"""
Make sure that each old schema has a valid upgrade path to the current one.
"""
-
+
for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
upgrader = UpgradeDatabaseSchemaService(None, None)
files = upgrader.scanForUpgradeFiles(dialect)
current_version = self._getSchemaVersion(upgrader.schemaLocation.child("current.sql"), "VERSION")
-
+
for child in upgrader.schemaLocation.child("old").child(dialect).globChildren("*.sql"):
old_version = self._getSchemaVersion(child, "VERSION")
upgrades = upgrader.determineUpgradeSequence(old_version, current_version, files, dialect)
@@ -115,7 +118,7 @@
# """
# Make sure that each upgrade file has a valid data upgrade file or None.
# """
-#
+#
# for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
# upgrader = UpgradeDatabaseSchemaService(None, None)
# files = upgrader.scanForUpgradeFiles(dialect)
@@ -124,6 +127,7 @@
# if result is not None:
# self.assertIsInstance(result, types.FunctionType)
+
@inlineCallbacks
def test_dbSchemaUpgrades(self):
"""
@@ -142,7 +146,7 @@
Use the postgres schema mechanism to do tests under a separate "namespace"
in postgres that we can quickly wipe clean afterwards.
"""
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
yield startTxn.execSQL(path.getContent())
@@ -150,21 +154,21 @@
@inlineCallbacks
def _loadVersion():
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
new_version = yield startTxn.execSQL("select value from calendarserver where name = 'VERSION';")
yield startTxn.commit()
returnValue(int(new_version[0][0]))
@inlineCallbacks
def _unloadOldSchema():
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldSchema():
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade;")
yield startTxn.commit()
@@ -174,6 +178,8 @@
test_upgrader = UpgradeDatabaseSchemaService(None, None)
expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"), "VERSION")
for child in test_upgrader.schemaLocation.child("old").child(POSTGRES_DIALECT).globChildren("*.sql"):
+
+ # Upgrade allowed
upgrader = UpgradeDatabaseSchemaService(store, None)
yield _loadOldSchema(child)
yield upgrader.databaseUpgrade()
@@ -182,13 +188,31 @@
self.assertEqual(new_version, expected_version)
+ # Upgrade disallowed
+ upgrader = UpgradeDatabaseSchemaService(store, None, failIfUpgradeNeeded=True, stopOnFail=False)
+ yield _loadOldSchema(child)
+ old_version = yield _loadVersion()
+ try:
+ yield upgrader.databaseUpgrade()
+ except RuntimeError:
+ pass
+ except Exception:
+ self.fail("RuntimeError not raised")
+ else:
+ self.fail("RuntimeError not raised")
+ new_version = yield _loadVersion()
+ yield _unloadOldSchema()
+
+ self.assertEqual(old_version, new_version)
+
+
@inlineCallbacks
def test_dbDataUpgrades(self):
"""
This does a full DB test of all possible data upgrade paths. For each old schema, it loads it into the DB
then runs the data upgrade service. This ensures all the upgrade_XX.py files work correctly - at least for
postgres.
-
+
TODO: this currently does not create any calendar data to test with. It simply runs the upgrade on an empty
store.
"""
@@ -203,7 +227,7 @@
Use the postgres schema mechanism to do tests under a separate "namespace"
in postgres that we can quickly wipe clean afterwards.
"""
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
yield startTxn.execSQL(path.getContent())
@@ -212,21 +236,21 @@
@inlineCallbacks
def _loadVersion():
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
new_version = yield startTxn.execSQL("select value from calendarserver where name = 'CALENDAR-DATAVERSION';")
yield startTxn.commit()
returnValue(int(new_version[0][0]))
@inlineCallbacks
def _unloadOldData():
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldData():
- startTxn = store.newTransaction("test_dbUpgrades")
+ startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade;")
yield startTxn.commit()
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py 2013-02-19 22:13:57 UTC (rev 10761)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py 2013-02-19 22:15:27 UTC (rev 10762)
@@ -52,7 +52,7 @@
"""
@classmethod
- def wrapService(cls, service, store, uid=None, gid=None):
+ def wrapService(cls, service, store, uid=None, gid=None, **kwargs):
"""
Create an L{UpgradeDatabaseSchemaService} when starting the database
so we can check the schema version and do any upgrades.
@@ -73,10 +73,10 @@
@return: a service
@rtype: L{IService}
"""
- return cls(store, service, uid=uid, gid=gid,)
+ return cls(store, service, uid=uid, gid=gid, **kwargs)
- def __init__(self, sqlStore, service, uid=None, gid=None):
+ def __init__(self, sqlStore, service, uid=None, gid=None, failIfUpgradeNeeded=False, stopOnFail=True):
"""
Initialize the service.
"""
@@ -84,6 +84,8 @@
self.sqlStore = sqlStore
self.uid = uid
self.gid = gid
+ self.failIfUpgradeNeeded = failIfUpgradeNeeded
+ self.stopOnFail = stopOnFail
self.schemaLocation = getModule(__name__).filePath.parent().parent().sibling("sql_schema")
self.pyLocation = getModule(__name__).filePath.parent()
@@ -118,6 +120,10 @@
)
self.log_error(msg)
raise RuntimeError(msg)
+ elif self.failIfUpgradeNeeded:
+ if self.stopOnFail:
+ reactor.stop()
+ raise RuntimeError("Database upgrade is needed but not allowed.")
else:
self.sqlStore.setUpgrading(True)
yield self.upgradeVersion(actual_version, required_version, dialect)
@@ -275,14 +281,14 @@
@type wrappedService: L{IService} or C{NoneType}
"""
- def __init__(self, sqlStore, service, uid=None, gid=None):
+ def __init__(self, sqlStore, service, **kwargs):
"""
Initialize the service.
@param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
@param service: Wrapped service. Can be C{None} when doing unit tests.
"""
- super(UpgradeDatabaseSchemaService, self).__init__(sqlStore, service, uid, gid)
+ super(UpgradeDatabaseSchemaService, self).__init__(sqlStore, service, **kwargs)
self.versionKey = "VERSION"
self.versionDescriptor = "schema"
@@ -328,14 +334,14 @@
@type wrappedService: L{IService} or C{NoneType}
"""
- def __init__(self, sqlStore, service, uid=None, gid=None):
+ def __init__(self, sqlStore, service, **kwargs):
"""
Initialize the service.
@param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
@param service: Wrapped service. Can be C{None} when doing unit tests.
"""
- super(UpgradeDatabaseDataService, self).__init__(sqlStore, service, uid, gid)
+ super(UpgradeDatabaseDataService, self).__init__(sqlStore, service, **kwargs)
self.versionKey = "CALENDAR-DATAVERSION"
self.versionDescriptor = "data"
@@ -380,14 +386,14 @@
@type wrappedService: L{IService} or C{NoneType}
"""
- def __init__(self, sqlStore, service, uid=None, gid=None):
+ def __init__(self, sqlStore, service, **kwargs):
"""
Initialize the service.
@param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
@param service: Wrapped service. Can be C{None} when doing unit tests.
"""
- super(UpgradeDatabaseOtherService, self).__init__(sqlStore, service, uid, gid)
+ super(UpgradeDatabaseOtherService, self).__init__(sqlStore, service, **kwargs)
self.versionDescriptor = "other upgrades"
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130219/5bec810b/attachment-0001.html>
More information about the calendarserver-changes
mailing list