[CalendarServer-changes] [15314] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Mon Nov 16 07:15:24 PST 2015
Revision: 15314
http://trac.calendarserver.org//changeset/15314
Author: cdaboo at apple.com
Date: 2015-11-16 07:15:23 -0800 (Mon, 16 Nov 2015)
Log Message:
-----------
Provide option to do schema check in calendarserver_upgrade tool. Provide better error reporting of failed upgrades.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tools/cmdline.py
CalendarServer/trunk/calendarserver/tools/upgrade.py
CalendarServer/trunk/requirements-cs.txt
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2015-11-16 15:15:23 UTC (rev 15314)
@@ -1556,7 +1556,8 @@
pps.addStep(
UpgradeDatabaseSchemaStep(
store, uid=overrideUID, gid=overrideGID,
- failIfUpgradeNeeded=config.FailIfUpgradeNeeded
+ failIfUpgradeNeeded=config.FailIfUpgradeNeeded,
+ checkExistingSchema=config.CheckExistingSchema,
)
)
Modified: CalendarServer/trunk/calendarserver/tools/cmdline.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/cmdline.py 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/calendarserver/tools/cmdline.py 2015-11-16 15:15:23 UTC (rev 15314)
@@ -29,9 +29,8 @@
from twisted.application.service import Service
from twisted.internet.defer import inlineCallbacks, succeed
-from twisted.logger import STDLibLogObserver
+from twisted.logger import STDLibLogObserver, FileLogObserver, formatEventAsClassicLogText
from twisted.python.logfile import LogFile
-from twisted.python.log import addObserver
import sys
from errno import ENOENT, EACCES
@@ -105,8 +104,9 @@
rotateLength=config.ErrorLogRotateMB * 1024 * 1024,
maxRotatedFiles=config.ErrorLogMaxRotatedFiles
)
- utilityLogObserver = Logger.makeFilteredFileLogObserver(utilityLogFile)
- addObserver(utilityLogObserver)
+ utilityLogObserver = FileLogObserver(utilityLogFile, lambda event: formatEventAsClassicLogText(event))
+ utilityLogObserver._encoding = "utf-8"
+ Logger.beginLoggingTo([utilityLogObserver, ])
config.ProcessType = "Utility"
config.UtilityServiceClass = _makeValidService
Modified: CalendarServer/trunk/calendarserver/tools/upgrade.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/upgrade.py 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/calendarserver/tools/upgrade.py 2015-11-16 15:15:23 UTC (rev 15314)
@@ -20,13 +20,15 @@
This tool allows any necessary upgrade to complete, then exits.
"""
-from __future__ import print_function
import os
import sys
import time
+from txdav.common.datastore.sql import CommonDataStore
+
from twisted.internet.defer import succeed
from twisted.logger import LogLevel, formatEvent
+from twisted.logger import FilteringLogObserver, LogLevelFilterPredicate
from twisted.python.text import wordWrap
from twisted.python.usage import Options, UsageError
@@ -77,6 +79,7 @@
optFlags = [
['status', 's', "Check database status and exit."],
+ ['check', 'c', "Check current database schema and exit."],
['postprocess', 'p', "Perform post-database-import processing."],
['debug', 'D', "Print log messages to STDOUT."],
]
@@ -146,6 +149,11 @@
# If we get this far the database is OK
if self.options["status"]:
self.output.write("Database OK.\n")
+ elif self.options["check"]:
+ if hasattr(CommonDataStore, "checkSchemaResults"):
+ self.output.write("Database check failed:\n{}\n".format("\n".join(CommonDataStore.checkSchemaResults))) #@UndefinedVariable
+ else:
+ self.output.write("Database check OK.\n")
else:
self.output.write("Upgrade complete, shutting down.\n")
UpgraderService.started = True
@@ -154,10 +162,15 @@
def doWorkWithoutStore(self):
"""
- Immediately stop. The upgrade will have been run before this.
+ Immediately stop. The upgrade will have been run before this and failed.
"""
if self.options["status"]:
self.output.write("Upgrade needed.\n")
+ elif self.options["check"]:
+ if hasattr(CommonDataStore, "checkSchemaResults"):
+ self.output.write("Database check failed:\n{}\n".format("\n".join(CommonDataStore.checkSchemaResults))) #@UndefinedVariable
+ else:
+ self.output.write("Database check OK.\n")
else:
self.output.write("Upgrade failed.\n")
UpgraderService.started = True
@@ -220,9 +233,12 @@
output.write(logDateString() + " " + text + "\n")
output.flush()
- if not options["status"]:
- log.levels().setLogLevelForNamespace(None, LogLevel.debug)
- log.observer.addObserver(onlyUpgradeEvents)
+ if not options["status"] and not options["check"]:
+ # When doing an upgrade always send L{LogLevel.warn} logging to the tool output
+ log.observer.addObserver(FilteringLogObserver(
+ onlyUpgradeEvents,
+ [LogLevelFilterPredicate(defaultLogLevel=LogLevel.warn), ]
+ ))
def customServiceMaker():
@@ -232,9 +248,12 @@
def _patchConfig(config):
- config.FailIfUpgradeNeeded = options["status"]
+ config.FailIfUpgradeNeeded = options["status"] or options["check"]
+ config.CheckExistingSchema = options["check"]
if options["prefix"]:
config.UpgradeHomePrefix = options["prefix"]
+ if not options["status"] and not options["check"]:
+ config.DefaultLogLevel = "debug"
def _onShutdown():
Modified: CalendarServer/trunk/requirements-cs.txt
===================================================================
--- CalendarServer/trunk/requirements-cs.txt 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/requirements-cs.txt 2015-11-16 15:15:23 UTC (rev 15314)
@@ -7,7 +7,7 @@
zope.interface==4.1.3
setuptools==18.5
- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@15306#egg=twextpy
+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@15312#egg=twextpy
cffi==1.3.0
pycparser==2.14
#twisted
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-11-16 15:15:23 UTC (rev 15314)
@@ -248,6 +248,10 @@
# tools from running if the database needs a schema
# upgrade.
+ "CheckExistingSchema": False, # Set to True to check the current database schema
+ # against the schema file matching the database schema
+ # version.
+
"UpgradeHomePrefix": "", # When upgrading, only upgrade homes where the owner UID starts with
# the specified prefix. The upgrade will only be partial and only
# apply to upgrade pieces that affect entire homes. The upgrade will
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2015-11-16 15:15:23 UTC (rev 15314)
@@ -62,6 +62,7 @@
from txdav.common.datastore.sql_apn import APNSubscriptionsMixin
from txdav.common.datastore.sql_directory import DelegatesAPIMixin, \
GroupsAPIMixin, GroupCacherAPIMixin
+from txdav.common.datastore.sql_dump import dumpSchema
from txdav.common.datastore.sql_imip import imipAPIMixin
from txdav.common.datastore.sql_notification import NotificationCollection
from txdav.common.datastore.sql_tables import _BIND_MODE_OWN, _BIND_STATUS_ACCEPTED, \
@@ -390,7 +391,33 @@
returnValue((False, None,))
+ @inlineCallbacks
+ def checkSchema(self, expected_schema, schema_name):
+ """
+ Check the schema actually in the database against the one in the supplied
+ schema file.
+ @param expected_schema: the expected schema
+ @type expected_schema: L{Schema}
+ """
+ txn = yield self.newTransaction(label="CommonDataStore.checkSchema")
+ try:
+ actual_schema = yield dumpSchema(txn, "actual", schema_name)
+ results = actual_schema.compare(expected_schema)
+ if results:
+ if not hasattr(self.__class__, "checkSchemaResults"):
+ self.__class__.checkSchemaResults = results
+ log.warn("Schema comparison mismatch:\n{}".format("\n".join(results)))
+ else:
+ log.warn("Schema comparison match")
+ except Exception as e:
+ log.error("Schema comparison match failed: {}".format(e))
+ yield txn.abort()
+ else:
+ yield txn.commit()
+
+
+
class TransactionStatsCollector(object):
"""
Used to log each SQL query and statistics about that query during the course of a single transaction.
@@ -990,7 +1017,11 @@
"""
for stmt in splitSQLString(sql):
if not stmt.startswith("--"):
- yield self.execSQL(stmt)
+ try:
+ yield self.execSQL(stmt)
+ except (RuntimeError, StandardError) as e:
+ e.stmt = "SQLBlock statement failed: {}".format(stmt)
+ raise
def commit(self):
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py 2015-11-15 18:17:50 UTC (rev 15313)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py 2015-11-16 15:15:23 UTC (rev 15314)
@@ -22,6 +22,7 @@
import re
+from twext.enterprise.dal.parseschema import schemaFromPath
from twext.python.log import Logger
from twisted.internet.defer import inlineCallbacks, returnValue
@@ -29,6 +30,8 @@
from twisted.python.modules import getModule
from twisted.python.reflect import namedObject
+from twistedcaldav.config import config
+
from txdav.common.datastore.upgrade.sql.others import attachment_migration
@@ -96,7 +99,7 @@
"""
log = Logger()
- def __init__(self, sqlStore, uid=None, gid=None, failIfUpgradeNeeded=False):
+ def __init__(self, sqlStore, uid=None, gid=None, failIfUpgradeNeeded=False, checkExistingSchema=False):
"""
Initialize the service.
"""
@@ -104,6 +107,7 @@
self.uid = uid
self.gid = gid
self.failIfUpgradeNeeded = failIfUpgradeNeeded
+ self.checkExistingSchema = checkExistingSchema
self.schemaLocation = getModule(__name__).filePath.parent().parent().sibling("sql_schema")
self.pyLocation = getModule(__name__).filePath.parent()
@@ -133,6 +137,14 @@
if required_version == actual_version:
self.log.warn("{vers} version check complete: no upgrade needed.", vers=self.versionDescriptor.capitalize())
+ if self.checkExistingSchema:
+ if dialect == "postgres-dialect":
+ expected_schema = self.schemaLocation.child("current.sql")
+ schema_name = "public"
+ else:
+ expected_schema = self.schemaLocation.child("current-oracle-dialect.sql")
+ schema_name = config.DatabaseConnection.user
+ yield self.sqlStore.checkSchema(schemaFromPath(expected_schema), schema_name)
elif required_version < actual_version:
msg = "Actual %s version %s is more recent than the expected version %s. The service cannot be started" % (
self.versionDescriptor, actual_version, required_version,
@@ -140,6 +152,13 @@
self.log.error(msg)
raise RuntimeError(msg)
elif self.failIfUpgradeNeeded:
+ if self.checkExistingSchema:
+ expected_schema = self.schemaLocation.child("old").child(dialect).child("v{}.sql".format(actual_version))
+ if dialect == "postgres-dialect":
+ schema_name = "public"
+ else:
+ schema_name = config.DatabaseConnection.user
+ yield self.sqlStore.checkSchema(schemaFromPath(expected_schema), schema_name)
raise NotAllowedToUpgrade()
else:
self.sqlStore.setUpgrading(True)
@@ -213,7 +232,7 @@
self.log.error("Database {vers} upgrade failed using: {path}", vers=self.versionDescriptor, path=fp.basename())
raise
- self.log.warn("{vers} upgraded from version {fr} to {o}.", vers=self.versionDescriptor.capitalize(), fr=fromVersion, to=toVersion)
+ self.log.warn("{vers} upgraded from version {fr} to {to}.", vers=self.versionDescriptor.capitalize(), fr=fromVersion, to=toVersion)
def getPathToUpgrades(self, dialect):
@@ -318,7 +337,13 @@
sql = fp.getContent()
yield sqlTxn.execSQLBlock(sql)
yield sqlTxn.commit()
- except RuntimeError:
+ except (RuntimeError, StandardError) as e:
+ if hasattr(e, "stmt"):
+ self.log.error("Apply upgrade failed for '{basename}' on statement: {stmt}\n{err}".format(
+ basename=fp.basename(),
+ stmt=e.stmt,
+ err=e,
+ ))
f = Failure()
yield sqlTxn.abort()
f.raiseException()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20151116/1ed253eb/attachment-0001.html>
More information about the calendarserver-changes
mailing list