[CalendarServer-changes] [2244] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Tue Mar 25 10:51:16 PDT 2008
Revision: 2244
http://trac.macosforge.org/projects/calendarserver/changeset/2244
Author: cdaboo at apple.com
Date: 2008-03-25 10:51:16 -0700 (Tue, 25 Mar 2008)
Log Message:
-----------
Upgrade the proxy DB with indexes if an older version is detected. This change involved change the base sql DB
behavior to allow persistent and non-persistent DB's to behave differently. Persistent DBs will call an "upgrade"
method when a version mismatch is detected, whereas non-persistent DBs will delete and recreate the index.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/admin/util.py
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/trunk/twistedcaldav/directory/digest.py
CalendarServer/trunk/twistedcaldav/directory/sqldb.py
CalendarServer/trunk/twistedcaldav/index.py
CalendarServer/trunk/twistedcaldav/sql.py
CalendarServer/trunk/twistedcaldav/test/test_sql.py
Added Paths:
-----------
CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py
Modified: CalendarServer/trunk/twistedcaldav/admin/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/util.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/admin/util.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -135,7 +135,7 @@
class EventCountingDatabase(AbstractSQLDatabase):
def __init__(self, fp):
- super(EventCountingDatabase, self).__init__(fp.path)
+ super(EventCountingDatabase, self).__init__(fp.path, False)
def _db_version(self):
return schema_version
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -313,11 +313,11 @@
dbType = "CALENDARUSERPROXY"
dbFilename = db_prefix + "calendaruserproxy"
- dbFormatVersion = "3"
+ dbFormatVersion = "4"
def __init__(self, path):
path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
- super(CalendarUserProxyDatabase, self).__init__(path)
+ super(CalendarUserProxyDatabase, self).__init__(path, True)
def setGroupMembers(self, principalUID, members):
"""
@@ -422,6 +422,27 @@
"""
)
+ def _db_upgrade_data_tables(self, q, old_version):
+ """
+ Upgrade the data from an older version of the DB.
+ @param q: a database cursor to use.
+ @param old_version: existing DB's version number
+ @type old_version: str
+ """
+
+ # Add index if old version is less than "4"
+ if int(old_version) < 4:
+ q.execute(
+ """
+ create index GROUPNAMES on GROUPS (GROUPNAME)
+ """
+ )
+ q.execute(
+ """
+ create index MEMBERS on GROUPS (MEMBER)
+ """
+ )
+
##
# Utilities
##
Modified: CalendarServer/trunk/twistedcaldav/directory/digest.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/digest.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/directory/digest.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -190,7 +190,7 @@
def __init__(self, path):
db_path = os.path.join(path, DigestCredentialsDB.dbFilename)
- super(DigestCredentialsDB, self).__init__(db_path, autocommit=False)
+ super(DigestCredentialsDB, self).__init__(db_path, False, autocommit=False)
self.exceptions = 0
def has_key(self, key):
Modified: CalendarServer/trunk/twistedcaldav/directory/sqldb.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/sqldb.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/directory/sqldb.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -60,7 +60,7 @@
def __init__(self, path):
path = os.path.join(path, SQLDirectoryManager.dbFilename)
- super(SQLDirectoryManager, self).__init__(path)
+ super(SQLDirectoryManager, self).__init__(path, True)
def loadFromXML(self, xmlFile):
parser = XMLAccountsParser(xmlFile)
Added: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py (rev 0)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -0,0 +1,173 @@
+##
+# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+
+import os
+
+import twistedcaldav.test.util
+
+#class ProxyPrincipalDB (twistedcaldav.test.util.TestCase):
+# """
+# Directory service provisioned principals.
+# """
+#
+# class old_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+#
+# def _db_version(self):
+# """
+# @return: the schema version assigned to this index.
+# """
+# return "3"
+#
+# def _db_init_data_tables(self, q):
+# """
+# Initialise the underlying database tables.
+# @param q: a database cursor to use.
+# """
+#
+# #
+# # GROUPS table
+# #
+# q.execute(
+# """
+# create table GROUPS (
+# GROUPNAME text,
+# MEMBER text
+# )
+# """
+# )
+#
+# class new_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+#
+# def _db_version(self):
+# """
+# @return: the schema version assigned to this index.
+# """
+# return "11"
+#
+# class newer_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+#
+# def _db_version(self):
+# """
+# @return: the schema version assigned to this index.
+# """
+# return "51"
+#
+# def test_normalDB(self):
+# """
+# DirectoryPrincipalResource.groupMembers()
+# """
+#
+# # Get the DB
+# db_path = self.mktemp()
+# os.mkdir(db_path)
+# db = CalendarUserProxyDatabase(db_path)
+# db.setGroupMembers("A", ("B", "C", "D",))
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+#
+# def test_DBIndexed(self):
+# """
+# DirectoryPrincipalResource.groupMembers()
+# """
+#
+# # Get the DB
+# db_path = self.mktemp()
+# os.mkdir(db_path)
+# db = CalendarUserProxyDatabase(db_path)
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+#
+# def test_OldDB(self):
+# """
+# DirectoryPrincipalResource.groupMembers()
+# """
+#
+# # Get the DB
+# db_path = self.mktemp()
+# os.mkdir(db_path)
+# db = self.old_CalendarUserProxyDatabase(db_path)
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+#
+# def test_DBUpgrade(self):
+# """
+# DirectoryPrincipalResource.groupMembers()
+# """
+#
+# # Get the DB
+# db_path = self.mktemp()
+# os.mkdir(db_path)
+# db = self.old_CalendarUserProxyDatabase(db_path)
+# db.setGroupMembers("A", ("B", "C", "D",))
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+# db._db_close()
+# db = None
+#
+# db = CalendarUserProxyDatabase(db_path)
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+# db._db_close()
+# db = None
+#
+# def test_DBUpgradeNewer(self):
+# """
+# DirectoryPrincipalResource.groupMembers()
+# """
+#
+# # Get the DB
+# db_path = self.mktemp()
+# os.mkdir(db_path)
+# db = self.old_CalendarUserProxyDatabase(db_path)
+# db.setGroupMembers("A", ("B", "C", "D",))
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+# db._db_close()
+# db = None
+#
+# db = self.new_CalendarUserProxyDatabase(db_path)
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+# db._db_close()
+# db = None
+#
+# def test_DBNoUpgradeNewer(self):
+# """
+# DirectoryPrincipalResource.groupMembers()
+# """
+#
+# # Get the DB
+# db_path = self.mktemp()
+# os.mkdir(db_path)
+# db = self.new_CalendarUserProxyDatabase(db_path)
+# db.setGroupMembers("A", ("B", "C", "D",))
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+# db._db_close()
+# db = None
+#
+# db = self.newer_CalendarUserProxyDatabase(db_path)
+# self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
+# self.assertEqual(db.getMemberships("B"), set(("A",)))
+# self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+# db._db_close()
+# db = None
+
+
\ No newline at end of file
Modified: CalendarServer/trunk/twistedcaldav/index.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/index.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/index.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -96,7 +96,7 @@
"""
self.resource = resource
db_filename = os.path.join(self.resource.fp.path, db_basename)
- super(AbstractCalendarIndex, self).__init__(db_filename)
+ super(AbstractCalendarIndex, self).__init__(db_filename, False)
def create(self):
"""
Modified: CalendarServer/trunk/twistedcaldav/sql.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/sql.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/sql.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -38,15 +38,19 @@
A generic SQL database.
"""
- def __init__(self, dbpath, autocommit=False):
+ def __init__(self, dbpath, persistent, autocommit=False):
"""
@param dbpath: the path where the db file is stored.
@type dbpath: str
+ @param persistent: C{True} if the data in the DB must be perserved during upgrades,
+ C{False} if the DB data can be re-created from an external source.
+ @type persistent: bool
@param autocommit: C{True} if auto-commit mode is desired, C{False} otherwise
@type autocommit: bool
"""
self.dbpath = dbpath
+ self.persistent = persistent
self.autocommit = autocommit
def _db_version(self):
@@ -104,20 +108,27 @@
if type is not None: type = type[0]
if (version != self._db_version()) or (type != self._db_type()):
+
+ # Clean-up first
+ q.close()
+ q = None
+ self._db_connection.close()
+ del(self._db_connection)
+
if version != self._db_version():
log.err("Database %s has different schema (v.%s vs. v.%s)"
% (db_filename, version, self._db_version()))
+
+ # Upgrade the DB
+ return self._db_upgrade(version)
+
if type != self._db_type():
log.err("Database %s has different type (%s vs. %s)"
% (db_filename, type, self._db_type()))
- # Delete this index and start over
- q.close()
- q = None
- self._db_connection.close()
- del(self._db_connection)
- os.remove(db_filename)
- return self._db()
+ # Delete this index and start over
+ os.remove(db_filename)
+ return self._db()
else:
self._db_init(db_filename, q)
@@ -203,6 +214,43 @@
"""
pass
+ def _db_upgrade(self, old_version):
+ """
+ Upgrade the database tables.
+ """
+
+ if self.persistent:
+ self._db_connection = sqlite.connect(self.dbpath, isolation_level=None)
+ q = self._db_connection.cursor()
+ self._db_upgrade_data_tables(q, old_version)
+ self._db_upgrade_schema(q)
+ self._db_close()
+ return self._db()
+ else:
+ # Non-persistent DB's by default can be removed and re-created. However, for simple
+ # DB upgrades they SHOULD override this method and handle those for better performance.
+ os.remove(self.dbpath)
+ return self._db()
+
+ def _db_upgrade_data_tables(self, q, old_version):
+ """
+ Upgrade the data from an older version of the DB.
+ """
+ # Persistent DB's MUST override this method and do a proper upgrade. Their data
+ # cannot be thrown away.
+ raise NotImplementedError("Persistent databases MUST support an upgrade method.")
+
+ def _db_upgrade_schema(self, q):
+ """
+ Upgrade the stored schema version to the current one.
+ """
+ q.execute(
+ """
+ insert or replace into CALDAV (KEY, VALUE)
+ values ('SCHEMA_VERSION', :1)
+ """, [self._db_version()]
+ )
+
def _db_close(self):
if hasattr(self, "_db_connection"):
self._db_connection.close()
Modified: CalendarServer/trunk/twistedcaldav/test/test_sql.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_sql.py 2008-03-25 17:48:03 UTC (rev 2243)
+++ CalendarServer/trunk/twistedcaldav/test/test_sql.py 2008-03-25 17:51:16 UTC (rev 2244)
@@ -29,14 +29,15 @@
class TestDB(AbstractSQLDatabase):
- def __init__(self, path, autocommit=False):
- super(SQL.TestDB, self).__init__(path, autocommit=autocommit)
+ def __init__(self, path, persistent=False, autocommit=False, version="1"):
+ self.version = version
+ super(SQL.TestDB, self).__init__(path, persistent, autocommit=autocommit)
def _db_version(self):
"""
@return: the schema version assigned to this index.
"""
- return 1
+ return self.version
def _db_type(self):
"""
@@ -62,6 +63,31 @@
"""
)
+ class TestDBRecreateUpgrade(TestDB):
+
+ class RecreateDBException(Exception):
+ pass
+ class UpgradeDBException(Exception):
+ pass
+
+ def __init__(self, path, persistent=False, autocommit=False):
+ super(SQL.TestDBRecreateUpgrade, self).__init__(path, persistent, autocommit=autocommit, version="2")
+
+ def _db_recreate(self):
+ raise self.RecreateDBException()
+
+ class TestDBCreateIndexOnUpgrade(TestDB):
+
+ def __init__(self, path, persistent=False, autocommit=False):
+ super(SQL.TestDBCreateIndexOnUpgrade, self).__init__(path, persistent, autocommit=autocommit, version="2")
+
+ def _db_upgrade_data_tables(self, q, old_version):
+ q.execute(
+ """
+ create index TESTING on TESTTYPE (VALUE)
+ """
+ )
+
class TestDBPauseInInit(TestDB):
def _db_init(self, db_filename, q):
@@ -182,3 +208,61 @@
self.assertTrue(t1.result)
self.assertTrue(t2.result)
+ def test_version_upgrade_nonpersistent(self):
+ """
+ Connect to database and create table
+ """
+ db = SQL.TestDB(self.mktemp(), autocommit=True)
+ self.assertTrue(db._db() is not None)
+ db._db_execute("INSERT into TESTTYPE (KEY, VALUE) values (:1, :2)", "FOO", "BAR")
+ items = db._db_execute("SELECT * from TESTTYPE")
+ self.assertEqual(items, [("FOO", "BAR")])
+ db._db_close()
+ db = None
+
+ db = SQL.TestDBRecreateUpgrade(self.mktemp(), autocommit=True)
+ self.assertRaises(SQL.TestDBRecreateUpgrade.RecreateDBException, db._db)
+ items = db._db_execute("SELECT * from TESTTYPE")
+ self.assertEqual(items, [])
+
+ def test_version_upgrade_persistent(self):
+ """
+ Connect to database and create table
+ """
+ db_file = self.mktemp()
+ db = SQL.TestDB(db_file, persistent=True, autocommit=True)
+ self.assertTrue(db._db() is not None)
+ db._db_execute("INSERT into TESTTYPE (KEY, VALUE) values (:1, :2)", "FOO", "BAR")
+ items = db._db_execute("SELECT * from TESTTYPE")
+ self.assertEqual(items, [("FOO", "BAR")])
+ db._db_close()
+ db = None
+
+ db = SQL.TestDBRecreateUpgrade(db_file, persistent=True, autocommit=True)
+ self.assertRaises(NotImplementedError, db._db)
+ self.assertTrue(os.path.exists(db_file))
+ db._db_close()
+ db = None
+
+ db = SQL.TestDB(db_file, persistent=True, autocommit=True)
+ self.assertTrue(db._db() is not None)
+ items = db._db_execute("SELECT * from TESTTYPE")
+ self.assertEqual(items, [("FOO", "BAR")])
+
+ def test_version_upgrade_persistent_add_index(self):
+ """
+ Connect to database and create table
+ """
+ db_file = self.mktemp()
+ db = SQL.TestDB(db_file, persistent=True, autocommit=True)
+ self.assertTrue(db._db() is not None)
+ db._db_execute("INSERT into TESTTYPE (KEY, VALUE) values (:1, :2)", "FOO", "BAR")
+ items = db._db_execute("SELECT * from TESTTYPE")
+ self.assertEqual(items, [("FOO", "BAR")])
+ db._db_close()
+ db = None
+
+ db = SQL.TestDBCreateIndexOnUpgrade(db_file, persistent=True, autocommit=True)
+ self.assertTrue(db._db() is not None)
+ items = db._db_execute("SELECT * from TESTTYPE")
+ self.assertEqual(items, [("FOO", "BAR")])
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080325/f47a214a/attachment-0001.html
More information about the calendarserver-changes
mailing list