[CalendarServer-changes] [1614] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jun 20 14:11:14 PDT 2007
Revision: 1614
http://trac.macosforge.org/projects/calendarserver/changeset/1614
Author: cdaboo at apple.com
Date: 2007-06-20 14:11:14 -0700 (Wed, 20 Jun 2007)
Log Message:
-----------
Fixing utf-8 issues. This involved switching the calendar index from returning utf-8 to returning unicode as
ideally we want to handle only unicode strings internally and only do utf-8 transcoding when data leaves our
domain (e.g. in http request/responses, files etc).
The sql database code was also refactored to remove a lot of duplicate code.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/caldavxml.py
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/trunk/twistedcaldav/directory/digest.py
CalendarServer/trunk/twistedcaldav/directory/sqldb.py
CalendarServer/trunk/twistedcaldav/index.py
CalendarServer/trunk/twistedcaldav/method/put_common.py
CalendarServer/trunk/twistedcaldav/sql.py
Removed Paths:
-------------
CalendarServer/trunk/twistedcaldav/db.py
Modified: CalendarServer/trunk/twistedcaldav/caldavxml.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/caldavxml.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/caldavxml.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -1136,7 +1136,7 @@
else:
values = item
- test = str(self)
+ test = unicode(str(self), "utf-8")
if self.caseless:
test = test.lower()
Deleted: CalendarServer/trunk/twistedcaldav/db.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/db.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/db.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -1,242 +0,0 @@
-##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# DRI: Cyrus Daboo, cdaboo at apple.com
-##
-
-"""
-Abstract SQLite Index base class. This class will be sub-classed for the
-different types of index we need in the server.
-"""
-
-__all__ = ["AbstractIndex"]
-
-import os
-
-try:
- import sqlite3 as sqlite
-except ImportError:
- from pysqlite2 import dbapi2 as sqlite
-
-from twisted.python import log
-
-db_basename = ".db.sqlite"
-
-class AbstractIndex(object):
- def __init__(self, resource, returnUTF8 = True):
- """
- @param resource: the L{twistedcaldav.static.CalDAVFile} resource to
- index.)
- """
- self.resource = resource
- self.utf8 = returnUTF8
-
- def _db_version(self):
- """
- @return: the schema version assigned to this index.
- """
- raise NotImplementedError
-
- def _db_type(self):
- """
- @return: the type assigned to this index.
- """
- raise NotImplementedError
-
- def _db(self):
- """
- Access the underlying database.
- @return: a db2 connection object for this index's underlying data store.
- """
- if not hasattr(self, "_db_connection"):
- db_filename = os.path.join(self.resource.fp.path, db_basename)
- self._db_connection = sqlite.connect(db_filename)
- if self.utf8:
- self._db_connection.text_factory = str
-
- #
- # Set up the schema
- #
- q = self._db_connection.cursor()
- try:
- # Create CALDAV table if needed
- q.execute(
- """
- select (1) from SQLITE_MASTER
- where TYPE = 'table' and NAME = 'CALDAV'
- """)
- caldav = q.fetchone()
-
- if caldav:
- q.execute(
- """
- select VALUE from CALDAV
- where KEY = 'SCHEMA_VERSION'
- """)
- version = q.fetchone()
-
- if version is not None: version = version[0]
-
- q.execute(
- """
- select VALUE from CALDAV
- where KEY = 'TYPE'
- """)
- type = q.fetchone()
-
- if type is not None: type = type[0]
-
- if (version != self._db_version()) or (type != self._db_type()):
- if version != self._db_version():
- log.err("Index %s has different schema (v.%s vs. v.%s)"
- % (db_filename, version, self._db_version()))
- if type != self._db_type():
- log.err("Index %s has different type (%s vs. %s)"
- % (db_filename, type, self._db_type()))
-
- # Delete this index and start over
- q.close()
- q = None
- self._db_connection.close()
- del(self._db_connection)
- os.remove(db_filename)
- return self._db()
- else:
- self._db_init(db_filename, q)
-
- self._db_connection.commit()
- finally:
- if q is not None: q.close()
- return self._db_connection
-
- def _db_init(self, db_filename, q):
- """
- Initialise the underlying database tables.
- @param db_filename: the file name of the index database.
- @param q: a database cursor to use.
- """
- log.msg("Initializing index %s" % (db_filename,))
-
- self._db_init_schema_table(q)
- self._db_init_data_tables(q)
- self._db_recreate()
-
- def _db_init_schema_table(self, q):
- """
- Initialise the underlying database tables.
- @param db_filename: the file name of the index database.
- @param q: a database cursor to use.
- """
-
- #
- # CALDAV table keeps track of our schema version and type
- #
- q.execute(
- """
- create table CALDAV (
- KEY text unique, VALUE text unique
- )
- """
- )
- q.execute(
- """
- insert into CALDAV (KEY, VALUE)
- values ('SCHEMA_VERSION', :1)
- """, [self._db_version()]
- )
- q.execute(
- """
- insert into CALDAV (KEY, VALUE)
- values ('TYPE', :1)
- """, [self._db_type()]
- )
-
- def _db_init_data_tables(self, q):
- """
- Initialise the underlying database tables.
- @param db_filename: the file name of the index database.
- @param q: a database cursor to use.
- """
- raise NotImplementedError
-
- def _db_recreate(self):
- """
- Recreate the database tables.
- """
- raise NotImplementedError
-
- def _add_to_db(self):
- """
- Add a record to the database.
- """
- raise NotImplementedError
-
- def _delete_from_db(self):
- """
- Delete a record from the database.
- """
- raise NotImplementedError
-
- def _db_values_for_sql(self, sql, *query_params):
- """
- Execute an SQL query and obtain the resulting values.
- @param sql: the SQL query to execute.
- @param query_params: parameters to C{sql}.
- @return: an interable of values in the first column of each row
- resulting from executing C{sql} with C{query_params}.
- @raise AssertionError: if the query yields multiple columns.
- """
- return (row[0] for row in self._db_execute(sql, *query_params))
-
- def _db_value_for_sql(self, sql, *query_params):
- """
- Execute an SQL query and obtain a single value.
- @param sql: the SQL query to execute.
- @param query_params: parameters to C{sql}.
- @return: the value resulting from the executing C{sql} with
- C{query_params}.
- @raise AssertionError: if the query yields multiple rows or columns.
- """
- value = None
- for row in self._db_values_for_sql(sql, *query_params):
- assert value is None, "Multiple values in DB for %s %s" % (sql, query_params)
- value = row
- return value
-
- def _db_execute(self, sql, *query_params):
- """
- Execute an SQL query and obtain the resulting values.
- @param sql: the SQL query to execute.
- @param query_params: parameters to C{sql}.
- @return: an interable of tuples for each row resulting from executing
- C{sql} with C{query_params}.
- """
- q = self._db().cursor()
- try:
- try:
- q.execute(sql, query_params)
- except:
- log.err("Exception while executing SQL: %r %r" % (sql, query_params))
- raise
- return q.fetchall()
- finally:
- q.close()
-
- def _db_commit (self):
- self._db_connection.commit()
-
- def _db_rollback(self):
- self._db_connection.rollback()
-
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -261,11 +261,11 @@
dbType = "CALENDARUSERPROXY"
dbFilename = ".db.calendaruserproxy"
- dbFormatVersion = "2"
+ dbFormatVersion = "3"
def __init__(self, path):
path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
- super(CalendarUserProxyDatabase, self).__init__(path, CalendarUserProxyDatabase.dbFormatVersion)
+ super(CalendarUserProxyDatabase, self).__init__(path)
def setGroupMembers(self, principalGUID, members):
"""
@@ -330,6 +330,12 @@
"""
self._db_execute("delete from GROUPS where GROUPNAME = :1", principalGUID)
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return CalendarUserProxyDatabase.dbFormatVersion
+
def _db_type(self):
"""
@return: the collection type assigned to this index.
Modified: CalendarServer/trunk/twistedcaldav/directory/digest.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/digest.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/directory/digest.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -149,13 +149,13 @@
dbType = "DIGESTCREDENTIALSCACHE"
dbFilename = ".db.digestcredentialscache"
- dbFormatVersion = "1"
+ dbFormatVersion = "2"
def __init__(self, path):
db_path = os.path.join(path, DigestCredentialsDB.dbFilename)
if os.path.exists(db_path):
os.remove(db_path)
- super(DigestCredentialsDB, self).__init__(db_path, DigestCredentialsDB.dbFormatVersion)
+ super(DigestCredentialsDB, self).__init__(db_path)
self.db = {}
def has_key(self, key):
@@ -230,6 +230,12 @@
"""
self._db_execute("delete from DIGESTCREDENTIALS where KEY = :1", key)
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return DigestCredentialsDB.dbFormatVersion
+
def _db_type(self):
"""
@return: the collection type assigned to this index.
Modified: CalendarServer/trunk/twistedcaldav/directory/sqldb.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/sqldb.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/directory/sqldb.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -57,11 +57,11 @@
"""
dbType = "DIRECTORYSERVICE"
dbFilename = ".db.accounts"
- dbFormatVersion = "2"
+ dbFormatVersion = "3"
def __init__(self, path):
path = os.path.join(path, SQLDirectoryManager.dbFilename)
- super(SQLDirectoryManager, self).__init__(path, SQLDirectoryManager.dbFormatVersion)
+ super(SQLDirectoryManager, self).__init__(path)
def loadFromXML(self, xmlFile):
parser = XMLAccountsParser(xmlFile)
@@ -209,6 +209,12 @@
self._db_execute("delete from GROUPS where MEMBER_SHORT_NAME = :1", shortName)
self._db_execute("delete from ADDRESSES where SHORT_NAME = :1", shortName)
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return SQLDirectoryManager.dbFormatVersion
+
def _db_type(self):
"""
@return: the collection type assigned to this index.
Modified: CalendarServer/trunk/twistedcaldav/index.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/index.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/index.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -24,13 +24,12 @@
"""
__all__ = [
- "AbstractIndex",
"Index",
"IndexSchedule",
]
-import os
import datetime
+import os
try:
import sqlite3 as sqlite
@@ -41,12 +40,13 @@
from twistedcaldav.ical import Component
from twistedcaldav.query import calendarquery
+from twistedcaldav.sql import AbstractSQLDatabase
from twistedcaldav import caldavxml
from vobject.icalendar import utc
db_basename = ".db.sqlite"
-schema_version = "4"
+schema_version = "5"
collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
#
@@ -79,7 +79,7 @@
which is not reserved.
"""
-class AbstractIndex(object):
+class AbstractCalendarIndex(AbstractSQLDatabase):
"""
Calendar collection index abstract base class that defines the apis for the index.
This will be subclassed for the two types of index behaviour we need: one for
@@ -93,6 +93,8 @@
C{resource.isPseudoCalendarCollection()} returns C{True}.)
"""
self.resource = resource
+ db_filename = os.path.join(self.resource.fp.path, db_basename)
+ super(AbstractCalendarIndex, self).__init__(db_filename)
def create(self):
"""
@@ -152,7 +154,8 @@
#
resources = []
for name in names:
- if name is not None and self.resource.getChild(name) is None:
+ name_utf8 = name.encode("utf-8")
+ if name is not None and self.resource.getChild(name_utf8) is None:
# Clean up
log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
self._delete_from_db(name, uid)
@@ -272,137 +275,19 @@
for row in rowiter:
name = row[0]
- if self.resource.getChild(name):
+ if self.resource.getChild(name.encode("utf-8")):
yield row
else:
log.err("Calendar resource %s is missing from %s. Removing from index."
% (name, self.resource))
self.deleteResource(name)
- def _db_type(self):
+ def _db_version(self):
"""
- @return: the collection type assigned to this index.
+ @return: the schema version assigned to this index.
"""
- raise NotImplementedError
+ return schema_version
- def _db(self):
- """
- Access the underlying database.
- @return: a db2 connection object for this index's underlying data store.
- """
- if not hasattr(self, "_db_connection"):
- db_filename = os.path.join(self.resource.fp.path, db_basename)
- try:
- self._db_connection = sqlite.connect(db_filename)
- except:
- log.err("Unable to open database file: %s" % (db_filename,))
- raise
-
- #
- # Set up the schema
- #
- q = self._db_connection.cursor()
- try:
- # Create CALDAV table if needed
- q.execute(
- """
- select (1) from SQLITE_MASTER
- where TYPE = 'table' and NAME = 'CALDAV'
- """)
- caldav = q.fetchone()
-
- if caldav:
- q.execute(
- """
- select VALUE from CALDAV
- where KEY = 'SCHEMA_VERSION'
- """)
- version = q.fetchone()
-
- if version is not None: version = version[0]
-
- q.execute(
- """
- select VALUE from CALDAV
- where KEY = 'TYPE'
- """)
- type = q.fetchone()
-
- if type is not None: type = type[0]
-
- if (version != schema_version) or (type != self._db_type()):
- if version != schema_version:
- log.err("Index %s has different schema (v.%s vs. v.%s)"
- % (db_filename, version, schema_version))
- if type != self._db_type():
- log.err("Index %s has different type (%s vs. %s)"
- % (db_filename, type, self._db_type()))
-
- # Delete this index and start over
- q.close()
- q = None
- self._db_connection.close()
- del(self._db_connection)
- os.remove(db_filename)
- return self._db()
-
- else:
- self._db_init(db_filename, q)
-
- self._db_connection.commit()
- finally:
- if q is not None: q.close()
- return self._db_connection
-
- def _db_init(self, db_filename, q):
- """
- Initialise the underlying database tables.
- @param db_filename: the file name of the index database.
- @param q: a database cursor to use.
- """
- log.msg("Initializing index %s" % (db_filename,))
-
- self._db_init_schema_table(q)
- self._db_init_data_tables(q)
-
- def _db_init_schema_table(self, q):
- """
- Initialise the underlying database tables.
- @param db_filename: the file name of the index database.
- @param q: a database cursor to use.
- """
-
- #
- # CALDAV table keeps track of our schema version and type
- #
- q.execute(
- """
- create table CALDAV (
- KEY text unique, VALUE text unique
- )
- """
- )
- q.execute(
- """
- insert into CALDAV (KEY, VALUE)
- values ('SCHEMA_VERSION', :1)
- """, [schema_version]
- )
- q.execute(
- """
- insert into CALDAV (KEY, VALUE)
- values ('TYPE', :1)
- """, [self._db_type()]
- )
-
- def _db_init_data_tables(self, q):
- """
- Initialise the underlying database tables.
- @param db_filename: the file name of the index database.
- @param q: a database cursor to use.
- """
- raise NotImplementedError
-
def _add_to_db(self, name, calendar, cursor = None):
"""
Records the given calendar resource in the index with the given name.
@@ -424,55 +309,7 @@
"""
raise NotImplementedError
- def _db_values_for_sql(self, sql, *query_params):
- """
- Execute an SQL query and obtain the resulting values.
- @param sql: the SQL query to execute.
- @param query_params: parameters to C{sql}.
- @return: an interable of values in the first column of each row
- resulting from executing C{sql} with C{query_params}.
- @raise AssertionError: if the query yields multiple columns.
- """
- return (row[0] for row in self._db_execute(sql, *query_params))
-
- def _db_value_for_sql(self, sql, *query_params):
- """
- Execute an SQL query and obtain a single value.
- @param sql: the SQL query to execute.
- @param query_params: parameters to C{sql}.
- @return: the value resulting from the executing C{sql} with
- C{query_params}.
- @raise AssertionError: if the query yields multiple rows or columns.
- """
- value = None
- for row in self._db_values_for_sql(sql, *query_params):
- assert value is None, "Multiple values in DB for %s %s" % (sql, query_params)
- value = row
- return value
-
- def _db_execute(self, sql, *query_params):
- """
- Execute an SQL query and obtain the resulting values.
- @param sql: the SQL query to execute.
- @param query_params: parameters to C{sql}.
- @return: an interable of tuples for each row resulting from executing
- C{sql} with C{query_params}.
- """
- q = self._db().cursor()
- try:
- try:
- q.execute(sql, query_params)
- except:
- log.err("Exception while executing SQL: %r %r" % (sql, query_params))
- raise
- return q.fetchall()
- finally:
- q.close()
-
- def _db_commit (self): self._db_connection.commit()
- def _db_rollback(self): self._db_connection.rollback()
-
-class CalendarIndex (AbstractIndex):
+class CalendarIndex (AbstractCalendarIndex):
"""
Calendar index - abstract class for indexer that indexes calendar objects in a collection.
"""
@@ -676,6 +513,11 @@
# Create database where the RESOURCE table has unique UID column.
self._db_init_data_tables_base(q, True)
+ def _db_recreate(self):
+ """
+ Re-create the database tables from existing calendar data.
+ """
+
#
# Populate the DB with data from already existing resources.
# This allows for index recovery if the DB file gets
@@ -782,6 +624,11 @@
# Create database where the RESOURCE table has a UID column that is not unique.
self._db_init_data_tables_base(q, False)
+ def _db_recreate(self):
+ """
+ Re-create the database tables from existing calendar data.
+ """
+
#
# Populate the DB with data from already existing resources.
# This allows for index recovery if the DB file gets
Modified: CalendarServer/trunk/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/put_common.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/method/put_common.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -348,7 +348,7 @@
if not result:
log.err(message)
raise HTTPError(ErrorResponse(responsecode.FORBIDDEN,
- NoUIDConflict(davxml.HRef.fromString(joinURL(parentForURL(destination_uri), rname)))
+ NoUIDConflict(davxml.HRef.fromString(joinURL(parentForURL(destination_uri), rname.encode("utf-8"))))
))
# Reserve UID
Modified: CalendarServer/trunk/twistedcaldav/sql.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/sql.py 2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/sql.py 2007-06-20 21:11:14 UTC (rev 1614)
@@ -38,15 +38,20 @@
A generic SQL database.
"""
- def __init__(self, dbpath, version):
+ def __init__(self, dbpath):
"""
- @param resource: the L{twistedcaldav.static.CalDAVFile} resource to
- index. C{resource} must be a calendar collection (ie.
- C{resource.isPseudoCalendarCollection()} returns C{True}.)
+
+ @param dbpath: the path where the db file is stored.
+ @type dbpath: str
"""
self.dbpath = dbpath
- self.version = version
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ raise NotImplementedError
+
def _db_type(self):
"""
@return: the collection type assigned to this index.
@@ -94,10 +99,10 @@
if type is not None: type = type[0]
- if (version != self.version) or (type != self._db_type()):
- if version != self.version:
+ if (version != self._db_version()) or (type != self._db_type()):
+ if version != self._db_version():
log.err("Database %s has different schema (v.%s vs. v.%s)"
- % (db_filename, version, self.version))
+ % (db_filename, version, self._db_version()))
if type != self._db_type():
log.err("Database %s has different type (%s vs. %s)"
% (db_filename, type, self._db_type()))
@@ -128,6 +133,7 @@
self._db_init_schema_table(q)
self._db_init_data_tables(q)
+ self._db_recreate()
def _db_init_schema_table(self, q):
"""
@@ -150,7 +156,7 @@
"""
insert into CALDAV (KEY, VALUE)
values ('SCHEMA_VERSION', :1)
- """, [self.version]
+ """, [self._db_version()]
)
q.execute(
"""
@@ -167,6 +173,12 @@
"""
raise NotImplementedError
+ def _db_recreate(self):
+ """
+ Recreate the database tables.
+ """
+ pass
+
def _db_values_for_sql(self, sql, *query_params):
"""
Execute an SQL query and obtain the resulting values.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20070620/263a10eb/attachment.html
More information about the calendarserver-changes
mailing list