[CalendarServer-changes] [1614] CalendarServer/trunk/twistedcaldav

source_changes at macosforge.org source_changes at macosforge.org
Wed Jun 20 14:11:14 PDT 2007


Revision: 1614
          http://trac.macosforge.org/projects/calendarserver/changeset/1614
Author:   cdaboo at apple.com
Date:     2007-06-20 14:11:14 -0700 (Wed, 20 Jun 2007)

Log Message:
-----------
Fixing utf-8 issues. This involved switching the calendar index from returning utf-8 to returning unicode as
ideally we want to handle only unicode strings internally and only do utf-8 transcoding when data leaves our
domain (e.g. in http request/responses, files etc).

The sql database code was also refactored to remove a lot of duplicate code.

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/caldavxml.py
    CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
    CalendarServer/trunk/twistedcaldav/directory/digest.py
    CalendarServer/trunk/twistedcaldav/directory/sqldb.py
    CalendarServer/trunk/twistedcaldav/index.py
    CalendarServer/trunk/twistedcaldav/method/put_common.py
    CalendarServer/trunk/twistedcaldav/sql.py

Removed Paths:
-------------
    CalendarServer/trunk/twistedcaldav/db.py

Modified: CalendarServer/trunk/twistedcaldav/caldavxml.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/caldavxml.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/caldavxml.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -1136,7 +1136,7 @@
         else:
             values = item
 
-        test = str(self)
+        test = unicode(str(self), "utf-8")
         if self.caseless:
             test = test.lower()
 

Deleted: CalendarServer/trunk/twistedcaldav/db.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/db.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/db.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -1,242 +0,0 @@
-##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# DRI: Cyrus Daboo, cdaboo at apple.com
-##
-
-"""
-Abstract SQLite Index base class. This class will be sub-classed for the
-different types of index we need in the server.
-"""
-
-__all__ = ["AbstractIndex"]
-
-import os
-
-try:
-    import sqlite3 as sqlite
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite
-
-from twisted.python import log
-
-db_basename = ".db.sqlite"
-
-class AbstractIndex(object):
-    def __init__(self, resource, returnUTF8 = True):
-        """
-        @param resource: the L{twistedcaldav.static.CalDAVFile} resource to
-            index.)
-        """
-        self.resource = resource
-        self.utf8 = returnUTF8
-
-    def _db_version(self):
-        """
-        @return: the schema version assigned to this index.
-        """
-        raise NotImplementedError
-        
-    def _db_type(self):
-        """
-        @return: the type assigned to this index.
-        """
-        raise NotImplementedError
-        
-    def _db(self):
-        """
-        Access the underlying database.
-        @return: a db2 connection object for this index's underlying data store.
-        """
-        if not hasattr(self, "_db_connection"):
-            db_filename = os.path.join(self.resource.fp.path, db_basename)
-            self._db_connection = sqlite.connect(db_filename)
-            if self.utf8:
-                self._db_connection.text_factory = str
-
-            #
-            # Set up the schema
-            #
-            q = self._db_connection.cursor()
-            try:
-                # Create CALDAV table if needed
-                q.execute(
-                    """
-                    select (1) from SQLITE_MASTER
-                     where TYPE = 'table' and NAME = 'CALDAV'
-                    """)
-                caldav = q.fetchone()
-
-                if caldav:
-                    q.execute(
-                        """
-                        select VALUE from CALDAV
-                         where KEY = 'SCHEMA_VERSION'
-                        """)
-                    version = q.fetchone()
-
-                    if version is not None: version = version[0]
-
-                    q.execute(
-                        """
-                        select VALUE from CALDAV
-                         where KEY = 'TYPE'
-                        """)
-                    type = q.fetchone()
-
-                    if type is not None: type = type[0]
-
-                    if (version != self._db_version()) or (type != self._db_type()):
-                        if version != self._db_version():
-                            log.err("Index %s has different schema (v.%s vs. v.%s)"
-                                    % (db_filename, version, self._db_version()))
-                        if type != self._db_type():
-                            log.err("Index %s has different type (%s vs. %s)"
-                                    % (db_filename, type, self._db_type()))
-
-                        # Delete this index and start over
-                        q.close()
-                        q = None
-                        self._db_connection.close()
-                        del(self._db_connection)
-                        os.remove(db_filename)
-                        return self._db()
-                else:
-                    self._db_init(db_filename, q)
-
-                self._db_connection.commit()
-            finally:
-                if q is not None: q.close()
-        return self._db_connection
-
-    def _db_init(self, db_filename, q):
-        """
-        Initialise the underlying database tables.
-        @param db_filename: the file name of the index database.
-        @param q:           a database cursor to use.
-        """
-        log.msg("Initializing index %s" % (db_filename,))
-
-        self._db_init_schema_table(q)
-        self._db_init_data_tables(q)
-        self._db_recreate()
-
-    def _db_init_schema_table(self, q):
-        """
-        Initialise the underlying database tables.
-        @param db_filename: the file name of the index database.
-        @param q:           a database cursor to use.
-        """
-
-        #
-        # CALDAV table keeps track of our schema version and type
-        #
-        q.execute(
-            """
-            create table CALDAV (
-                KEY text unique, VALUE text unique
-            )
-            """
-        )
-        q.execute(
-            """
-            insert into CALDAV (KEY, VALUE)
-            values ('SCHEMA_VERSION', :1)
-            """, [self._db_version()]
-        )
-        q.execute(
-            """
-            insert into CALDAV (KEY, VALUE)
-            values ('TYPE', :1)
-            """, [self._db_type()]
-        )
-
-    def _db_init_data_tables(self, q):
-        """
-        Initialise the underlying database tables.
-        @param db_filename: the file name of the index database.
-        @param q:           a database cursor to use.
-        """
-        raise NotImplementedError
-
-    def _db_recreate(self):
-        """
-        Recreate the database tables.
-        """
-        raise NotImplementedError
-
-    def _add_to_db(self):
-        """
-        Add a record to the database.
-        """
-        raise NotImplementedError
-    
-    def _delete_from_db(self):
-        """
-        Delete a record from the database.
-        """
-        raise NotImplementedError
-    
-    def _db_values_for_sql(self, sql, *query_params):
-        """
-        Execute an SQL query and obtain the resulting values.
-        @param sql: the SQL query to execute.
-        @param query_params: parameters to C{sql}.
-        @return: an interable of values in the first column of each row
-            resulting from executing C{sql} with C{query_params}.
-        @raise AssertionError: if the query yields multiple columns.
-        """
-        return (row[0] for row in self._db_execute(sql, *query_params))
-
-    def _db_value_for_sql(self, sql, *query_params):
-        """
-        Execute an SQL query and obtain a single value.
-        @param sql: the SQL query to execute.
-        @param query_params: parameters to C{sql}.
-        @return: the value resulting from the executing C{sql} with
-            C{query_params}.
-        @raise AssertionError: if the query yields multiple rows or columns.
-        """
-        value = None
-        for row in self._db_values_for_sql(sql, *query_params):
-            assert value is None, "Multiple values in DB for %s %s" % (sql, query_params)
-            value = row
-        return value
-
-    def _db_execute(self, sql, *query_params):
-        """
-        Execute an SQL query and obtain the resulting values.
-        @param sql: the SQL query to execute.
-        @param query_params: parameters to C{sql}.
-        @return: an interable of tuples for each row resulting from executing
-            C{sql} with C{query_params}.
-        """
-        q = self._db().cursor()
-        try:
-            try:
-                q.execute(sql, query_params)
-            except:
-                log.err("Exception while executing SQL: %r %r" % (sql, query_params))
-                raise
-            return q.fetchall()
-        finally:
-            q.close()
-
-    def _db_commit  (self):
-        self._db_connection.commit()
-
-    def _db_rollback(self):
-        self._db_connection.rollback()
-

Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -261,11 +261,11 @@
     
     dbType = "CALENDARUSERPROXY"
     dbFilename = ".db.calendaruserproxy"
-    dbFormatVersion = "2"
+    dbFormatVersion = "3"
 
     def __init__(self, path):
         path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
-        super(CalendarUserProxyDatabase, self).__init__(path, CalendarUserProxyDatabase.dbFormatVersion)
+        super(CalendarUserProxyDatabase, self).__init__(path)
 
     def setGroupMembers(self, principalGUID, members):
         """
@@ -330,6 +330,12 @@
         """
         self._db_execute("delete from GROUPS where GROUPNAME = :1", principalGUID)
     
+    def _db_version(self):
+        """
+        @return: the schema version assigned to this index.
+        """
+        return CalendarUserProxyDatabase.dbFormatVersion
+        
     def _db_type(self):
         """
         @return: the collection type assigned to this index.

Modified: CalendarServer/trunk/twistedcaldav/directory/digest.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/digest.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/directory/digest.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -149,13 +149,13 @@
     
     dbType = "DIGESTCREDENTIALSCACHE"
     dbFilename = ".db.digestcredentialscache"
-    dbFormatVersion = "1"
+    dbFormatVersion = "2"
 
     def __init__(self, path):
         db_path = os.path.join(path, DigestCredentialsDB.dbFilename)
         if os.path.exists(db_path):
             os.remove(db_path)
-        super(DigestCredentialsDB, self).__init__(db_path, DigestCredentialsDB.dbFormatVersion)
+        super(DigestCredentialsDB, self).__init__(db_path)
         self.db = {}
     
     def has_key(self, key):
@@ -230,6 +230,12 @@
         """
         self._db_execute("delete from DIGESTCREDENTIALS where KEY = :1", key)
     
+    def _db_version(self):
+        """
+        @return: the schema version assigned to this index.
+        """
+        return DigestCredentialsDB.dbFormatVersion
+        
     def _db_type(self):
         """
         @return: the collection type assigned to this index.

Modified: CalendarServer/trunk/twistedcaldav/directory/sqldb.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/sqldb.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/directory/sqldb.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -57,11 +57,11 @@
     """
     dbType = "DIRECTORYSERVICE"
     dbFilename = ".db.accounts"
-    dbFormatVersion = "2"
+    dbFormatVersion = "3"
 
     def __init__(self, path):
         path = os.path.join(path, SQLDirectoryManager.dbFilename)
-        super(SQLDirectoryManager, self).__init__(path, SQLDirectoryManager.dbFormatVersion)
+        super(SQLDirectoryManager, self).__init__(path)
 
     def loadFromXML(self, xmlFile):
         parser = XMLAccountsParser(xmlFile)
@@ -209,6 +209,12 @@
         self._db_execute("delete from GROUPS    where MEMBER_SHORT_NAME = :1", shortName)
         self._db_execute("delete from ADDRESSES where SHORT_NAME        = :1", shortName)
     
+    def _db_version(self):
+        """
+        @return: the schema version assigned to this index.
+        """
+        return SQLDirectoryManager.dbFormatVersion
+        
     def _db_type(self):
         """
         @return: the collection type assigned to this index.

Modified: CalendarServer/trunk/twistedcaldav/index.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/index.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/index.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -24,13 +24,12 @@
 """
 
 __all__ = [
-    "AbstractIndex",
     "Index",
     "IndexSchedule",
 ]
 
-import os
 import datetime
+import os
 
 try:
     import sqlite3 as sqlite
@@ -41,12 +40,13 @@
 
 from twistedcaldav.ical import Component
 from twistedcaldav.query import calendarquery
+from twistedcaldav.sql import AbstractSQLDatabase
 from twistedcaldav import caldavxml
 
 from vobject.icalendar import utc
 
 db_basename = ".db.sqlite"
-schema_version = "4"
+schema_version = "5"
 collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
 
 #
@@ -79,7 +79,7 @@
     which is not reserved.
     """
 
-class AbstractIndex(object):
+class AbstractCalendarIndex(AbstractSQLDatabase):
     """
     Calendar collection index abstract base class that defines the apis for the index.
     This will be subclassed for the two types of index behaviour we need: one for
@@ -93,6 +93,8 @@
             C{resource.isPseudoCalendarCollection()} returns C{True}.)
         """
         self.resource = resource
+        db_filename = os.path.join(self.resource.fp.path, db_basename)
+        super(AbstractCalendarIndex, self).__init__(db_filename)
 
     def create(self):
         """
@@ -152,7 +154,8 @@
         #
         resources = []
         for name in names:
-            if name is not None and self.resource.getChild(name) is None:
+            name_utf8 = name.encode("utf-8")
+            if name is not None and self.resource.getChild(name_utf8) is None:
                 # Clean up
                 log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
                 self._delete_from_db(name, uid)
@@ -272,137 +275,19 @@
             
         for row in rowiter:
             name = row[0]
-            if self.resource.getChild(name):
+            if self.resource.getChild(name.encode("utf-8")):
                 yield row
             else:
                 log.err("Calendar resource %s is missing from %s. Removing from index."
                         % (name, self.resource))
                 self.deleteResource(name)
 
-    def _db_type(self):
+    def _db_version(self):
         """
-        @return: the collection type assigned to this index.
+        @return: the schema version assigned to this index.
         """
-        raise NotImplementedError
+        return schema_version
         
-    def _db(self):
-        """
-        Access the underlying database.
-        @return: a db2 connection object for this index's underlying data store.
-        """
-        if not hasattr(self, "_db_connection"):
-            db_filename = os.path.join(self.resource.fp.path, db_basename)
-            try:
-                self._db_connection = sqlite.connect(db_filename)
-            except:
-                log.err("Unable to open database file: %s" % (db_filename,))
-                raise
-
-            #
-            # Set up the schema
-            #
-            q = self._db_connection.cursor()
-            try:
-                # Create CALDAV table if needed
-                q.execute(
-                    """
-                    select (1) from SQLITE_MASTER
-                     where TYPE = 'table' and NAME = 'CALDAV'
-                    """)
-                caldav = q.fetchone()
-
-                if caldav:
-                    q.execute(
-                        """
-                        select VALUE from CALDAV
-                         where KEY = 'SCHEMA_VERSION'
-                        """)
-                    version = q.fetchone()
-
-                    if version is not None: version = version[0]
-
-                    q.execute(
-                        """
-                        select VALUE from CALDAV
-                         where KEY = 'TYPE'
-                        """)
-                    type = q.fetchone()
-
-                    if type is not None: type = type[0]
-
-                    if (version != schema_version) or (type != self._db_type()):
-                        if version != schema_version:
-                            log.err("Index %s has different schema (v.%s vs. v.%s)"
-                                    % (db_filename, version, schema_version))
-                        if type != self._db_type():
-                            log.err("Index %s has different type (%s vs. %s)"
-                                    % (db_filename, type, self._db_type()))
-
-                        # Delete this index and start over
-                        q.close()
-                        q = None
-                        self._db_connection.close()
-                        del(self._db_connection)
-                        os.remove(db_filename)
-                        return self._db()
-
-                else:
-                    self._db_init(db_filename, q)
-
-                self._db_connection.commit()
-            finally:
-                if q is not None: q.close()
-        return self._db_connection
-
-    def _db_init(self, db_filename, q):
-        """
-        Initialise the underlying database tables.
-        @param db_filename: the file name of the index database.
-        @param q:           a database cursor to use.
-        """
-        log.msg("Initializing index %s" % (db_filename,))
-
-        self._db_init_schema_table(q)
-        self._db_init_data_tables(q)
-
-    def _db_init_schema_table(self, q):
-        """
-        Initialise the underlying database tables.
-        @param db_filename: the file name of the index database.
-        @param q:           a database cursor to use.
-        """
-
-        #
-        # CALDAV table keeps track of our schema version and type
-        #
-        q.execute(
-            """
-            create table CALDAV (
-                KEY text unique, VALUE text unique
-            )
-            """
-        )
-        q.execute(
-            """
-            insert into CALDAV (KEY, VALUE)
-            values ('SCHEMA_VERSION', :1)
-            """, [schema_version]
-        )
-        q.execute(
-            """
-            insert into CALDAV (KEY, VALUE)
-            values ('TYPE', :1)
-            """, [self._db_type()]
-        )
-
-    def _db_init_data_tables(self, q):
-        """
-        Initialise the underlying database tables.
-        @param db_filename: the file name of the index database.
-        @param q:           a database cursor to use.
-        """
-        raise NotImplementedError
-
     def _add_to_db(self, name, calendar, cursor = None):
         """
         Records the given calendar resource in the index with the given name.
@@ -424,55 +309,7 @@
         """
         raise NotImplementedError
     
-    def _db_values_for_sql(self, sql, *query_params):
-        """
-        Execute an SQL query and obtain the resulting values.
-        @param sql: the SQL query to execute.
-        @param query_params: parameters to C{sql}.
-        @return: an interable of values in the first column of each row
-            resulting from executing C{sql} with C{query_params}.
-        @raise AssertionError: if the query yields multiple columns.
-        """
-        return (row[0] for row in self._db_execute(sql, *query_params))
-
-    def _db_value_for_sql(self, sql, *query_params):
-        """
-        Execute an SQL query and obtain a single value.
-        @param sql: the SQL query to execute.
-        @param query_params: parameters to C{sql}.
-        @return: the value resulting from the executing C{sql} with
-            C{query_params}.
-        @raise AssertionError: if the query yields multiple rows or columns.
-        """
-        value = None
-        for row in self._db_values_for_sql(sql, *query_params):
-            assert value is None, "Multiple values in DB for %s %s" % (sql, query_params)
-            value = row
-        return value
-
-    def _db_execute(self, sql, *query_params):
-        """
-        Execute an SQL query and obtain the resulting values.
-        @param sql: the SQL query to execute.
-        @param query_params: parameters to C{sql}.
-        @return: an interable of tuples for each row resulting from executing
-            C{sql} with C{query_params}.
-        """
-        q = self._db().cursor()
-        try:
-            try:
-                q.execute(sql, query_params)
-            except:
-                log.err("Exception while executing SQL: %r %r" % (sql, query_params))
-                raise
-            return q.fetchall()
-        finally:
-            q.close()
-
-    def _db_commit  (self): self._db_connection.commit()
-    def _db_rollback(self): self._db_connection.rollback()
-
-class CalendarIndex (AbstractIndex):
+class CalendarIndex (AbstractCalendarIndex):
     """
     Calendar index - abstract class for indexer that indexes calendar objects in a collection.
     """
@@ -676,6 +513,11 @@
         # Create database where the RESOURCE table has unique UID column.
         self._db_init_data_tables_base(q, True)
 
+    def _db_recreate(self):
+        """
+        Re-create the database tables from existing calendar data.
+        """
+        
         #
         # Populate the DB with data from already existing resources.
         # This allows for index recovery if the DB file gets
@@ -782,6 +624,11 @@
         # Create database where the RESOURCE table has a UID column that is not unique.
         self._db_init_data_tables_base(q, False)
 
+    def _db_recreate(self):
+        """
+        Re-create the database tables from existing calendar data.
+        """
+        
         #
         # Populate the DB with data from already existing resources.
         # This allows for index recovery if the DB file gets

Modified: CalendarServer/trunk/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/put_common.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/method/put_common.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -348,7 +348,7 @@
                 if not result:
                     log.err(message)
                     raise HTTPError(ErrorResponse(responsecode.FORBIDDEN,
-                        NoUIDConflict(davxml.HRef.fromString(joinURL(parentForURL(destination_uri), rname)))
+                        NoUIDConflict(davxml.HRef.fromString(joinURL(parentForURL(destination_uri), rname.encode("utf-8"))))
                     ))
             
             # Reserve UID

Modified: CalendarServer/trunk/twistedcaldav/sql.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/sql.py	2007-06-20 21:06:28 UTC (rev 1613)
+++ CalendarServer/trunk/twistedcaldav/sql.py	2007-06-20 21:11:14 UTC (rev 1614)
@@ -38,15 +38,20 @@
     A generic SQL database.
     """
 
-    def __init__(self, dbpath, version):
+    def __init__(self, dbpath):
         """
-        @param resource: the L{twistedcaldav.static.CalDAVFile} resource to
-            index. C{resource} must be a calendar collection (ie.
-            C{resource.isPseudoCalendarCollection()} returns C{True}.)
+        
+        @param dbpath: the path where the db file is stored.
+        @type dbpath: str
         """
         self.dbpath = dbpath
-        self.version = version
 
+    def _db_version(self):
+        """
+        @return: the schema version assigned to this index.
+        """
+        raise NotImplementedError
+        
     def _db_type(self):
         """
         @return: the collection type assigned to this index.
@@ -94,10 +99,10 @@
 
                     if type is not None: type = type[0]
 
-                    if (version != self.version) or (type != self._db_type()):
-                        if version != self.version:
+                    if (version != self._db_version()) or (type != self._db_type()):
+                        if version != self._db_version():
                             log.err("Database %s has different schema (v.%s vs. v.%s)"
-                                    % (db_filename, version, self.version))
+                                    % (db_filename, version, self._db_version()))
                         if type != self._db_type():
                             log.err("Database %s has different type (%s vs. %s)"
                                     % (db_filename, type, self._db_type()))
@@ -128,6 +133,7 @@
 
         self._db_init_schema_table(q)
         self._db_init_data_tables(q)
+        self._db_recreate()
 
     def _db_init_schema_table(self, q):
         """
@@ -150,7 +156,7 @@
             """
             insert into CALDAV (KEY, VALUE)
             values ('SCHEMA_VERSION', :1)
-            """, [self.version]
+            """, [self._db_version()]
         )
         q.execute(
             """
@@ -167,6 +173,12 @@
         """
         raise NotImplementedError
 
+    def _db_recreate(self):
+        """
+        Recreate the database tables.
+        """
+        pass
+
     def _db_values_for_sql(self, sql, *query_params):
         """
         Execute an SQL query and obtain the resulting values.

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20070620/263a10eb/attachment.html


More information about the calendarserver-changes mailing list