[CalendarServer-changes] [4558] CalendarServer/branches/users/cdaboo/partition-4464
source_changes at macosforge.org
source_changes at macosforge.org
Mon Sep 28 13:37:17 PDT 2009
Revision: 4558
http://trac.macosforge.org/projects/calendarserver/changeset/4558
Author: cdaboo at apple.com
Date: 2009-09-28 13:37:17 -0700 (Mon, 28 Sep 2009)
Log Message:
-----------
Changed proxy DB over to using ADBAPI class. Added support for PostgreSQL as an ADBAPI database.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py
CalendarServer/branches/users/cdaboo/partition-4464/conf/auth/accounts.dtd
CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-apple.plist
CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-test.plist
CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd.plist
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/database.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/augment.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxyloader.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_augment.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipaldb.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/upgrade.py
Modified: CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -65,35 +65,35 @@
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "support"))
from version import version as getVersion
version = "%s (%s)" % getVersion()
-from twistedcaldav.log import Logger, LoggingMixIn
-from twistedcaldav.log import logLevelForNamespace, setLogLevelForNamespace
+from twistedcaldav import memcachepool
+from twistedcaldav.accesslog import AMPCommonAccessLoggingObserver
+from twistedcaldav.accesslog import AMPLoggingFactory
from twistedcaldav.accesslog import DirectoryLogWrapperResource
from twistedcaldav.accesslog import RotatingFileAccessLoggingObserver
-from twistedcaldav.accesslog import AMPLoggingFactory
-from twistedcaldav.accesslog import AMPCommonAccessLoggingObserver
-from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
+from twistedcaldav.config import ConfigurationError
from twistedcaldav.config import config
-from twistedcaldav.config import ConfigurationError
-from twistedcaldav.resource import CalDAVResource, AuthenticationWrapper
-from twistedcaldav.directory import augment
+from twistedcaldav.directory import augment, calendaruserproxy
+from twistedcaldav.directory.aggregate import AggregateDirectoryService
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
from twistedcaldav.directory.digest import QopDigestCredentialFactory
from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
-from twistedcaldav.directory.aggregate import AggregateDirectoryService
from twistedcaldav.directory.sudo import SudoDirectoryService
from twistedcaldav.directory.util import NotFilePath
from twistedcaldav.directory.wiki import WikiDirectoryService
+from twistedcaldav.localization import processLocalizationFiles
+from twistedcaldav.log import Logger, LoggingMixIn
+from twistedcaldav.log import logLevelForNamespace, setLogLevelForNamespace
+from twistedcaldav.mail import IMIPReplyInboxResource
+from twistedcaldav.notify import installNotificationClient
+from twistedcaldav.pdmonster import PDClientAddressWrapper
+from twistedcaldav.resource import CalDAVResource, AuthenticationWrapper
from twistedcaldav.static import CalendarHomeProvisioningFile
from twistedcaldav.static import IScheduleInboxFile
from twistedcaldav.static import TimezoneServiceFile
-from twistedcaldav.mail import IMIPReplyInboxResource
+from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twistedcaldav.timezones import TimezoneCache
from twistedcaldav.upgrade import upgradeData
-from twistedcaldav.pdmonster import PDClientAddressWrapper
-from twistedcaldav import memcachepool
-from twistedcaldav.notify import installNotificationClient
from twistedcaldav.util import getNCPU
-from twistedcaldav.localization import processLocalizationFiles
try:
from twistedcaldav.authkerb import NegotiateCredentialFactory
@@ -509,6 +509,19 @@
raise
#
+ # Setup the PoxyDB Service
+ #
+ proxydbClass = namedClass(config.ProxyDBService.type)
+
+ self.log_info("Configuring proxydb service of type: %s" % (proxydbClass,))
+
+ try:
+ calendaruserproxy.ProxyDBService = proxydbClass(**config.ProxyDBService.params)
+ except IOError, e:
+ self.log_error("Could not start proxydb service")
+ raise
+
+ #
# Make sure proxies get initialized
#
if config.ProxyLoadFromFile:
@@ -516,7 +529,7 @@
loader = XMLCalendarUserProxyLoader(config.ProxyLoadFromFile)
return loader.updateProxyDB()
- reactor.addSystemEventTrigger("before", "startup", _doProxyUpdate)
+ reactor.addSystemEventTrigger("after", "startup", _doProxyUpdate)
#
# Configure Memcached Client Pool
Modified: CalendarServer/branches/users/cdaboo/partition-4464/conf/auth/accounts.dtd
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/conf/auth/accounts.dtd 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/conf/auth/accounts.dtd 2009-09-28 20:37:17 UTC (rev 4558)
@@ -17,10 +17,10 @@
<!ELEMENT accounts (user*, group*, resource*, location*) >
<!ATTLIST accounts realm CDATA "">
- <!ELEMENT user (uid*, guid, password?, name?, first-name?, last-name?, email-address*,)>
+ <!ELEMENT user (uid*, guid, password?, name?, first-name?, last-name?, email-address*)>
<!ATTLIST user repeat CDATA "1">
- <!ELEMENT group (uid*, guid, password?, name?, members, enable-calendar?,)>
+ <!ELEMENT group (uid*, guid, password?, name?, members)>
<!ATTLIST group repeat CDATA "1">
<!ELEMENT resource (uid*, guid, password?, name?,)>
Modified: CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-apple.plist 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-apple.plist 2009-09-28 20:37:17 UTC (rev 4558)
@@ -198,7 +198,54 @@
</dict>
-->
+ <!-- PostgreSQL Augment Service -->
<!--
+ <key>AugmentService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.augment.AugmentPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>augments</string>
+ </dict>
+ </dict>
+ -->
+
+ <!-- Sqlite ProxyDB Service -->
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxySqliteDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>dbpath</key>
+ <string>/etc/caldavd/proxies.sqlite</string>
+ </dict>
+ </dict>
+
+ <!-- PostgreSQL ProxyDB Service -->
+ <!--
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>proxies</string>
+ </dict>
+ </dict>
+ -->
+
+ <!--
Special principals
These principals are granted special access and/or perform
Modified: CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-test.plist 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd-test.plist 2009-09-28 20:37:17 UTC (rev 4558)
@@ -190,6 +190,53 @@
</dict>
-->
+ <!-- PostgreSQL Augment Service -->
+ <!--
+ <key>AugmentService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.augment.AugmentPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>augments</string>
+ </dict>
+ </dict>
+ -->
+
+ <!-- Sqlite ProxyDB Service -->
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxySqliteDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>dbpath</key>
+ <string>data/proxies.sqlite</string>
+ </dict>
+ </dict>
+
+ <!-- PostgreSQL ProxyDB Service -->
+ <!--
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>proxies</string>
+ </dict>
+ </dict>
+ -->
+
<key>ProxyLoadFromFile</key>
<string>conf/auth/proxies-test.xml</string>
Modified: CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd.plist 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/conf/caldavd.plist 2009-09-28 20:37:17 UTC (rev 4558)
@@ -193,7 +193,54 @@
</dict>
-->
+ <!-- PostgreSQL Augment Service -->
<!--
+ <key>AugmentService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.augment.AugmentPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>augments</string>
+ </dict>
+ </dict>
+ -->
+
+ <!-- Sqlite ProxyDB Service -->
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxySqliteDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>dbpath</key>
+ <string>/etc/caldavd/proxies.sqlite</string>
+ </dict>
+ </dict>
+
+ <!-- PostgreSQL ProxyDB Service -->
+ <!--
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>proxies</string>
+ </dict>
+ </dict>
+ -->
+
+ <!--
Special principals
These principals are granted special access and/or perform
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/database.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/database.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/database.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -14,14 +14,21 @@
# limitations under the License.
##
-from twistedcaldav.log import Logger
-
from twisted.enterprise.adbapi import ConnectionPool
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.python.threadpool import ThreadPool
+from twistedcaldav.config import ConfigurationError
+from twistedcaldav.log import Logger
+
import thread
+try:
+ import pgdb
+except:
+ pgdb = None
+#pgdb = None
+
"""
Generic ADAPI database access object.
"""
@@ -133,6 +140,14 @@
self.initialized = False
@inlineCallbacks
+ def clean(self):
+
+ if not self.initialized:
+ yield self.open()
+
+ yield self._db_empty_data_tables()
+
+ @inlineCallbacks
def execute(self, sql, *query_params):
if not self.initialized:
@@ -187,13 +202,8 @@
"""
raise NotImplementedError
- @inlineCallbacks
def _test_schema_table(self):
- result = (yield self._db_value_for_sql("""
- select (1) from SQLITE_MASTER
- where TYPE = 'table' and NAME = 'CALDAV'
- """))
- returnValue(result)
+ return self._test_table("CALDAV")
@inlineCallbacks
def _db_init(self):
@@ -221,16 +231,13 @@
#
# CALDAV table keeps track of our schema version and type
#
+ yield self._create_table("CALDAV", (
+ ("KEY", "text unique"),
+ ("VALUE", "text unique"),
+ ), True)
+
yield self._db_execute(
"""
- create table if not exists CALDAV (
- KEY text unique,
- VALUE text unique
- )
- """
- )
- yield self._db_execute(
- """
insert or ignore into CALDAV (KEY, VALUE)
values ('SCHEMA_VERSION', :1)
""", (self._db_version(),)
@@ -248,6 +255,14 @@
"""
raise NotImplementedError
+ def _db_empty_data_tables(self):
+ """
+ Delete the database tables.
+ """
+
+ # Implementations can override this to re-create data
+ pass
+
def _db_recreate(self):
"""
Recreate the database tables.
@@ -318,6 +333,7 @@
@raise AssertionError: if the query yields multiple columns.
"""
+ sql = self._prepare_statement(sql)
results = (yield self.pool.runQuery(sql, *query_params))
returnValue(tuple(results))
@@ -333,6 +349,7 @@
@raise AssertionError: if the query yields multiple columns.
"""
+ sql = self._prepare_statement(sql)
results = (yield self.pool.runQuery(sql, *query_params))
returnValue(tuple([row[0] for row in results]))
@@ -363,4 +380,178 @@
C{sql} with C{query_params}.
"""
+ sql = self._prepare_statement(sql)
return self.pool.runOperation(sql, *query_params)
+
+ """
+ Since different databases support different types of columns and modifiers on those we need to
+ have an "abstract" way of specifying columns in our code and then map the abstract specifiers to
+ the underlying DB's allowed types.
+
+ Types we can use are:
+
+ integer
+ text
+ text(n)
+ date
+ serial
+
+ The " unique" modifier can be appended to any of those.
+ """
+ def _map_column_types(self, type):
+ raise NotImplementedError
+
+ def _create_table(self, name, columns, ifnotexists=False):
+ raise NotImplementedError
+
+ def _test_table(self, name):
+ raise NotImplementedError
+
+ def _prepare_statement(self, sql):
+ raise NotImplementedError
+
+class ADBAPISqliteMixin(object):
+
+ @classmethod
+ def _map_column_types(self, coltype):
+
+ result = ""
+ splits = coltype.split()
+ if splits[0] == "integer":
+ result = "integer"
+ elif splits[0] == "text":
+ result = "text"
+ elif splits[0].startswith("text("):
+ result = splits[0]
+ elif splits[0] == "date":
+ result = "date"
+ elif splits[0] == "serial":
+ result = "integer primary key autoincrement"
+
+ if len(splits) > 1 and splits[1] == "unique":
+ result += " unique"
+
+ return result
+
+ @inlineCallbacks
+ def _create_table(self, name, columns, ifnotexists=False):
+
+ colDefs = ["%s %s" % (colname, self._map_column_types(coltype)) for colname, coltype in columns]
+ statement = "create table %s%s (%s)" % (
+ "if not exists " if ifnotexists else "",
+ name,
+ ", ".join(colDefs),
+ )
+ yield self._db_execute(statement)
+
+ @inlineCallbacks
+ def _test_table(self, name):
+ result = (yield self._db_value_for_sql("""
+ select (1) from SQLITE_MASTER
+ where TYPE = 'table' and NAME = '%s'
+ """ % (name,)))
+ returnValue(result)
+
+ def _prepare_statement(self, sql):
+ # We are going to use the sqlite syntax of :1, :2 etc for our
+ # internal statements so we do not need to remap those
+ return sql
+
+if pgdb:
+
+ class ADBAPIPostgreSQLMixin(object):
+
+ @classmethod
+ def _map_column_types(self, coltype):
+
+ result = ""
+ splits = coltype.split()
+ if splits[0] == "integer":
+ result = "integer"
+ elif splits[0] == "text":
+ result = "text"
+ elif splits[0].startswith("text("):
+ result = "char" + splits[0][4:]
+ elif splits[0] == "date":
+ result = "date"
+ elif splits[0] == "serial":
+ result = "serial"
+
+ if len(splits) > 1 and splits[1] == "unique":
+ result += " unique"
+
+ return result
+
+ @inlineCallbacks
+ def _create_table(self, name, columns, ifnotexists=False):
+
+ colDefs = ["%s %s" % (colname, self._map_column_types(coltype)) for colname, coltype in columns]
+ statement = "create table %s (%s)" % (
+ name,
+ ", ".join(colDefs),
+ )
+
+ try:
+ yield self._db_execute(statement)
+ except pgdb.DatabaseError:
+
+ if not ifnotexists:
+ raise
+
+ result = (yield self._test_table(name))
+ if not result:
+ raise
+
+ @inlineCallbacks
+ def _test_table(self, name):
+ result = (yield self._db_value_for_sql("""
+ select * from pg_tables
+ where tablename = '%s'
+ """ % (name.lower(),)))
+ returnValue(result)
+
+ @inlineCallbacks
+ def _db_init_schema_table(self):
+ """
+ Initialise the underlying database tables.
+ @param db_filename: the file name of the index database.
+ @param q: a database cursor to use.
+ """
+
+ #
+ # CALDAV table keeps track of our schema version and type
+ #
+ try:
+ yield self._create_table("CALDAV", (
+ ("KEY", "text unique"),
+ ("VALUE", "text unique"),
+ ), True)
+
+ yield self._db_execute(
+ """
+ insert into CALDAV (KEY, VALUE)
+ values ('SCHEMA_VERSION', :1)
+ """, (self._db_version(),)
+ )
+ yield self._db_execute(
+ """
+ insert into CALDAV (KEY, VALUE)
+ values ('TYPE', :1)
+ """, (self._db_type(),)
+ )
+ except pgdb.DatabaseError:
+ pass
+
+ def _prepare_statement(self, sql):
+ # Convert :1, :2 etc format into %s
+ ctr = 1
+ while sql.find(":%d" % (ctr,)) != -1:
+ sql = sql.replace(":%d" % (ctr,), "%s")
+ ctr += 1
+ return sql
+
+else:
+ class ADBAPIPostgreSQLMixin(object):
+
+ def __init__(self):
+ raise ConfigurationError("PostgreSQL module not available.")
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/augment.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/augment.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/augment.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -16,7 +16,8 @@
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
-from twistedcaldav.database import AbstractADBAPIDatabase
+from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\
+ ADBAPIPostgreSQLMixin
from twistedcaldav.directory.xmlaugmentsparser import XMLAugmentsParser
import time
@@ -243,43 +244,48 @@
@inlineCallbacks
def _db_init_data_tables(self):
"""
- Initialise the underlying database tables.
+ Initialize the underlying database tables.
"""
#
# TESTTYPE table
#
- yield self._db_execute(
- """
- create table AUGMENTS (
- GUID text unique,
- ENABLED text(1),
- PARTITIONID text,
- CALENDARING text(1),
- AUTOSCHEDULE text(1),
- CUADDRS text
- )
- """
- )
- yield self._db_execute(
- """
- create table PARTITIONS (
- PARTITIONID integer primary key autoincrement,
- HOSTEDAT text
- )
- """
- )
+ yield self._create_table("AUGMENTS", (
+ ("GUID", "text unique"),
+ ("ENABLED", "text(1)"),
+ ("PARTITIONID", "text"),
+ ("CALENDARING", "text(1)"),
+ ("AUTOSCHEDULE", "text(1)"),
+ ("CUADDRS", "text"),
+ ))
+ yield self._create_table("PARTITIONS", (
+ ("PARTITIONID", "serial"),
+ ("HOSTEDAT", "text"),
+ ))
+
@inlineCallbacks
- def _db_remove_data_tables(self):
- yield self._db_execute("drop table if exists AUGMENTS")
- yield self._db_execute("drop table if exists PARTITIONS")
+ def _db_empty_data_tables(self):
+ yield self._db_execute("delete from AUGMENTS")
+ yield self._db_execute("delete from PARTITIONS")
-class AugmentSqliteDB(AugmentADAPI):
+class AugmentSqliteDB(ADBAPISqliteMixin, AugmentADAPI):
"""
Sqlite based augment database implementation.
"""
def __init__(self, dbpath):
- super(AugmentSqliteDB, self).__init__("Augments", "sqlite3", (dbpath,))
+ ADBAPISqliteMixin.__init__(self)
+ AugmentADAPI.__init__(self, "Augments", "sqlite3", (dbpath,))
+
+class AugmentPostgreSQLDB(ADBAPIPostgreSQLMixin, AugmentADAPI):
+ """
+ PostgreSQL based augment database implementation.
+ """
+
+ def __init__(self, host, database):
+
+ ADBAPIPostgreSQLMixin.__init__(self)
+ AugmentADAPI.__init__(self, "Augments", "pgdb", (), host=host, database=database,)
+
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -20,7 +20,10 @@
__all__ = [
"CalendarUserProxyPrincipalResource",
- "CalendarUserProxyDatabase",
+ "ProxyDB",
+ "ProxyDBService",
+ "ProxySqliteDB",
+ "ProxyPostgreSQLDB",
]
from twisted.internet.defer import returnValue
@@ -33,16 +36,16 @@
from twisted.web2.dav.noneprops import NonePropertyStore
from twistedcaldav.config import config
+from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\
+ ADBAPIPostgreSQLMixin
from twistedcaldav.extensions import DAVFile, DAVPrincipalResource
from twistedcaldav.extensions import ReadOnlyWritePropertiesResourceMixIn
from twistedcaldav.memcacher import Memcacher
from twistedcaldav.resource import CalDAVComplianceMixIn
from twistedcaldav.directory.util import NotFilePath
-from twistedcaldav.sql import AbstractSQLDatabase, db_prefix
from twistedcaldav.log import LoggingMixIn
import itertools
-import os
import time
class PermissionsMixIn (ReadOnlyWritePropertiesResourceMixIn):
@@ -121,14 +124,10 @@
"""
Return the SQL database for this group principal.
- @return: the L{CalendarUserProxyDatabase} for the principal collection.
+ @return: the L{ProxyDB} for the principal collection.
"""
+ return ProxyDBService
- # The db is located in the principal collection root
- if not hasattr(self.pcollection, "calendar_user_proxy_db"):
- setattr(self.pcollection, "calendar_user_proxy_db", CalendarUserProxyDatabase(config.DataRoot))
- return self.pcollection.calendar_user_proxy_db
-
def resourceType(self):
if self.proxyType == "calendar-proxy-read":
return davxml.ResourceType.calendarproxyread
@@ -360,7 +359,7 @@
memberships = yield self._index().getMemberships(self.uid)
returnValue([p for p in [self.pcollection.principalForUID(uid) for uid in memberships] if p])
-class CalendarUserProxyDatabase(AbstractSQLDatabase, LoggingMixIn):
+class ProxyDB(AbstractADBAPIDatabase, LoggingMixIn):
"""
A database to maintain calendar user proxy group memberships.
@@ -372,11 +371,9 @@
"""
- dbType = "CALENDARUSERPROXY"
- dbFilename = "calendaruserproxy.sqlite"
- dbOldFilename = db_prefix + "calendaruserproxy"
- dbFormatVersion = "4"
-
+ schema_version = "4"
+ schema_type = "ProxyDB"
+
class ProxyDBMemcacher(Memcacher):
def setMembers(self, guid, members):
@@ -442,11 +439,10 @@
theTime = int(time.time())
return theTime
- def __init__(self, path):
- path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
- super(CalendarUserProxyDatabase, self).__init__(path, True)
+ def __init__(self, dbID, dbapiName, dbapiArgs, **kwargs):
+ AbstractADBAPIDatabase.__init__(self, dbID, dbapiName, dbapiArgs, True, **kwargs)
- self._memcacher = CalendarUserProxyDatabase.ProxyDBMemcacher("proxyDB")
+ self._memcacher = ProxyDB.ProxyDBMemcacher("proxyDB")
@inlineCallbacks
def setGroupMembers(self, principalUID, members):
@@ -463,17 +459,19 @@
current_members = ()
current_members = set(current_members)
- self.setGroupMembersInDatabase(principalUID, members)
-
- # Update cache
+ # Find changes
update_members = set(members)
-
remove_members = current_members.difference(update_members)
add_members = update_members.difference(current_members)
+
+ yield self.changeGroupMembersInDatabase(principalUID, add_members, remove_members)
+
+ # Update cache
for member in itertools.chain(remove_members, add_members,):
_ignore = yield self._memcacher.deleteMembership(member)
_ignore = yield self._memcacher.deleteMember(principalUID)
+ @inlineCallbacks
def setGroupMembersInDatabase(self, principalUID, members):
"""
A blocking call to add a group membership record in the database.
@@ -482,11 +480,25 @@
@param members: a list UIDs of principals that are members of this group.
"""
# Remove what is there, then add it back.
- self._delete_from_db(principalUID)
- self._add_to_db(principalUID, members)
- self._db_commit()
+ yield self._delete_from_db(principalUID)
+ yield self._add_to_db(principalUID, members)
@inlineCallbacks
+ def changeGroupMembersInDatabase(self, principalUID, addMembers, removeMembers):
+ """
+ A blocking call to add a group membership record in the database.
+
+ @param principalUID: the UID of the group principal to add.
+ @param addMembers: a list UIDs of principals to be added as members of this group.
+ @param removeMembers: a list UIDs of principals to be removed as members of this group.
+ """
+ # Remove what is there, then add it back.
+ for member in removeMembers:
+ yield self._delete_from_db_one(principalUID, member)
+ for member in addMembers:
+ yield self._add_to_db_one(principalUID, member)
+
+ @inlineCallbacks
def removeGroup(self, principalUID):
"""
Remove a group membership record.
@@ -497,8 +509,7 @@
# Need to get the members before we do the delete
members = yield self.getMembers(principalUID)
- self._delete_from_db(principalUID)
- self._db_commit()
+ yield self._delete_from_db(principalUID)
# Update cache
if members:
@@ -540,7 +551,7 @@
for suffix in ("calendar-proxy-read", "calendar-proxy-write",):
groupUID = "%s#%s" % (principalUID, suffix,)
- self._delete_from_db(groupUID)
+ yield self._delete_from_db(groupUID)
# Update cache
members = yield self.getMembers(groupUID)
@@ -553,9 +564,8 @@
for groupUID in memberships:
yield self._memcacher.deleteMember(groupUID)
- self._delete_from_db_member(principalUID)
+ yield self._delete_from_db_member(principalUID)
yield self._memcacher.deleteMembership(principalUID)
- self._db_commit()
self._memcacher.clearDeletionTimer(principalUID)
@inlineCallbacks
@@ -566,13 +576,15 @@
@return: a deferred returning a C{set} of members.
"""
+ @inlineCallbacks
def _members():
- return set([row[0] for row in self._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", principalUID)])
+ result = set([row[0] for row in (yield self.query("select MEMBER from GROUPS where GROUPNAME = :1", (principalUID,)))])
+ returnValue(result)
# Pull from cache
result = yield self._memcacher.getMembers(principalUID)
if result is None:
- result = _members()
+ result = (yield _members())
yield self._memcacher.setMembers(principalUID, result)
returnValue(result)
@@ -584,16 +596,19 @@
@return: a deferred returning a C{set} of memberships.
"""
+ @inlineCallbacks
def _members():
- return set([row[0] for row in self._db_execute("select GROUPNAME from GROUPS where MEMBER = :1", principalUID)])
+ result = set([row[0] for row in (yield self.query("select GROUPNAME from GROUPS where MEMBER = :1", (principalUID,)))])
+ returnValue(result)
# Pull from cache
result = yield self._memcacher.getMemberships(principalUID)
if result is None:
- result = _members()
+ result = (yield _members())
yield self._memcacher.setMemberships(principalUID, result)
returnValue(result)
+ @inlineCallbacks
def _add_to_db(self, principalUID, members):
"""
Insert the specified entry into the database.
@@ -602,42 +617,66 @@
@param members: a list of UIDs or principals that are members of this group.
"""
for member in members:
- self._db_execute(
+ yield self.execute(
"""
insert into GROUPS (GROUPNAME, MEMBER)
values (:1, :2)
- """, principalUID, member
+ """, (principalUID, member,)
)
+ def _add_to_db_one(self, principalUID, memberUID):
+ """
+ Insert the specified entry into the database.
+
+ @param principalUID: the UID of the group principal to add.
+ @param memberUID: the UID of the principal that is being added as a member of this group.
+ """
+ return self.execute(
+ """
+ insert into GROUPS (GROUPNAME, MEMBER)
+ values (:1, :2)
+ """, (principalUID, memberUID,)
+ )
+
def _delete_from_db(self, principalUID):
"""
Deletes the specified entry from the database.
@param principalUID: the UID of the group principal to remove.
"""
- self._db_execute("delete from GROUPS where GROUPNAME = :1", principalUID)
+ return self.execute("delete from GROUPS where GROUPNAME = :1", (principalUID,))
+ def _delete_from_db_one(self, principalUID, memberUID):
+ """
+ Deletes the specified entry from the database.
+
+ @param principalUID: the UID of the group principal to remove.
+ @param memberUID: the UID of the principal that is being removed as a member of this group.
+ """
+ return self.execute("delete from GROUPS where GROUPNAME = :1 and MEMBER = :2", (principalUID, memberUID,))
+
def _delete_from_db_member(self, principalUID):
"""
Deletes the specified member entry from the database.
@param principalUID: the UID of the member principal to remove.
"""
- self._db_execute("delete from GROUPS where MEMBER = :1", principalUID)
+ return self.execute("delete from GROUPS where MEMBER = :1", (principalUID,))
def _db_version(self):
"""
@return: the schema version assigned to this index.
"""
- return CalendarUserProxyDatabase.dbFormatVersion
+ return ProxyDB.schema_version
def _db_type(self):
"""
@return: the collection type assigned to this index.
"""
- return CalendarUserProxyDatabase.dbType
+ return ProxyDB.schema_type
- def _db_init_data_tables(self, q):
+ @inlineCallbacks
+ def _db_init_data_tables(self):
"""
Initialise the underlying database tables.
@param q: a database cursor to use.
@@ -646,48 +685,90 @@
#
# GROUPS table
#
- q.execute(
+ yield self._create_table("GROUPS", (
+ ("GROUPNAME", "text"),
+ ("MEMBER", "text"),
+ ))
+
+ yield self._db_execute(
"""
- create table GROUPS (
- GROUPNAME text,
- MEMBER text
- )
- """
- )
- q.execute(
- """
create index GROUPNAMES on GROUPS (GROUPNAME)
"""
)
- q.execute(
+ yield self._db_execute(
"""
create index MEMBERS on GROUPS (MEMBER)
"""
)
- def _db_upgrade_data_tables(self, q, old_version):
+ @inlineCallbacks
+ def _db_upgrade_data_tables(self, old_version):
"""
Upgrade the data from an older version of the DB.
- @param q: a database cursor to use.
@param old_version: existing DB's version number
@type old_version: str
"""
# Add index if old version is less than "4"
if int(old_version) < 4:
- q.execute(
+ yield self._db_execute(
"""
create index GROUPNAMES on GROUPS (GROUPNAME)
"""
)
- q.execute(
+ yield self._db_execute(
"""
create index MEMBERS on GROUPS (MEMBER)
"""
)
+ def _db_empty_data_tables(self):
+ """
+ Empty the underlying database tables.
+ @param q: a database cursor to use.
+ """
+ #
+ # GROUPS table
+ #
+ return self._db_execute("delete from GROUPS")
+ @inlineCallbacks
+ def clean(self):
+
+ if not self.initialized:
+ yield self.open()
+
+ for group in [row[0] for row in (yield self.query("select GROUPNAME from GROUPS"))]:
+ self.removeGroup(group)
+
+ yield super(ProxyDB, self).clean()
+
+
+ProxyDBService = None # Global proxyDB service
+
+
+class ProxySqliteDB(ADBAPISqliteMixin, ProxyDB):
+ """
+ Sqlite based proxy database implementation.
+ """
+
+ def __init__(self, dbpath):
+
+ ADBAPISqliteMixin.__init__(self)
+ ProxyDB.__init__(self, "Proxies", "sqlite3", (dbpath,))
+
+class ProxyPostgreSQLDB(ADBAPIPostgreSQLMixin, ProxyDB):
+ """
+ PostgreSQL based augment database implementation.
+ """
+
+ def __init__(self, host, database):
+
+ ADBAPIPostgreSQLMixin.__init__(self, )
+ ProxyDB.__init__(self, "Proxies", "pgdb", (), host=host, database=database,)
+
+
##
# Utilities
##
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxyloader.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxyloader.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxyloader.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -16,8 +16,7 @@
from xml.etree.ElementTree import ElementTree
from xml.parsers.expat import ExpatError
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
-from twistedcaldav.config import config
+from twistedcaldav.directory import calendaruserproxy
from twisted.internet.defer import inlineCallbacks
import types
@@ -129,7 +128,7 @@
@inlineCallbacks
def updateProxyDB(self):
- db = CalendarUserProxyDatabase(config.DataRoot)
+ db = calendaruserproxy.ProxyDBService
for item in self.items:
guid, write_proxies, read_proxies = item
for proxy in write_proxies:
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -49,7 +49,7 @@
from twistedcaldav.config import config
from twistedcaldav.cache import DisabledCacheNotifier, PropfindCacheMixin
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+from twistedcaldav.directory import calendaruserproxy
from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyPrincipalResource
from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
from twistedcaldav.directory.util import NotFilePath
@@ -641,16 +641,11 @@
"""
Return the SQL database for calendar user proxies.
- @return: the L{CalendarUserProxyDatabase} for the principal collection.
+ @return: the L{ProxyDB} for the principal collection.
"""
- # Get the principal collection we are contained in
- pcollection = self.parent.parent
-
# The db is located in the principal collection root
- if not hasattr(pcollection, "calendar_user_proxy_db"):
- setattr(pcollection, "calendar_user_proxy_db", CalendarUserProxyDatabase(config.DataRoot))
- return pcollection.calendar_user_proxy_db
+ return calendaruserproxy.ProxyDBService
def alternateURIs(self):
# FIXME: Add API to IDirectoryRecord for getting a record URI?
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_augment.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_augment.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_augment.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -15,7 +15,8 @@
##
from twistedcaldav.test.util import TestCase
-from twistedcaldav.directory.augment import AugmentXMLDB, AugmentSqliteDB
+from twistedcaldav.directory.augment import AugmentXMLDB, AugmentSqliteDB,\
+ AugmentPostgreSQLDB
from twisted.internet.defer import inlineCallbacks
from twistedcaldav.directory.xmlaugmentsparser import XMLAugmentsParser
import cStringIO
@@ -106,3 +107,30 @@
yield self._checkRecord(db, item)
yield self._checkNoRecord(db, "D11F03A0-97EA-48AF-9A6C-FAC7F3975767")
+
+class AugmentPostgreSQLTests(AugmentTests):
+
+ @inlineCallbacks
+ def test_read(self):
+
+ db = AugmentPostgreSQLDB("localhost", "augments")
+ yield db.clean()
+
+ dbxml = AugmentXMLDB((xmlFile,))
+ for record in dbxml.db.values():
+ yield db.addAugmentRecord(record)
+
+ for item in testRecords:
+ yield self._checkRecord(db, item)
+
+ yield self._checkNoRecord(db, "D11F03A0-97EA-48AF-9A6C-FAC7F3975767")
+
+try:
+ import pgdb
+except ImportError:
+ AugmentPostgreSQLTests.skip = True
+else:
+ try:
+ db = pgdb.connect(host="localhost", database="augments")
+ except:
+ AugmentPostgreSQLTests.skip = True
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipaldb.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipaldb.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipaldb.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -15,18 +15,19 @@
##
from twistedcaldav.config import config
-import os
-
from twisted.internet.defer import inlineCallbacks
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+from twistedcaldav.directory.calendaruserproxy import ProxySqliteDB,\
+ ProxyPostgreSQLDB
import twistedcaldav.test.util
+from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
+from twistedcaldav.directory import calendaruserproxy
-class ProxyPrincipalDB (twistedcaldav.test.util.TestCase):
+class ProxyPrincipalDBSqlite (twistedcaldav.test.util.TestCase):
"""
Directory service provisioned principals.
"""
- class old_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+ class old_ProxyDB(ProxySqliteDB):
def _db_version(self):
"""
@@ -34,7 +35,7 @@
"""
return "3"
- def _db_init_data_tables(self, q):
+ def _db_init_data_tables(self):
"""
Initialise the underlying database tables.
@param q: a database cursor to use.
@@ -43,7 +44,7 @@
#
# GROUPS table
#
- q.execute(
+ return self.execute(
"""
create table GROUPS (
GROUPNAME text,
@@ -52,7 +53,7 @@
"""
)
- class new_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+ class new_ProxyDB(ProxySqliteDB):
def _db_version(self):
"""
@@ -60,7 +61,7 @@
"""
return "11"
- class newer_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+ class newer_ProxyDB(ProxySqliteDB):
def _db_version(self):
"""
@@ -73,8 +74,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -83,29 +83,28 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
+ @inlineCallbacks
def test_DBIndexed(self):
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+ db = ProxySqliteDB(db_path)
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ @inlineCallbacks
def test_OldDB(self):
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.old_CalendarUserProxyDatabase(db_path)
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+ db = self.old_ProxyDB(db_path)
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set())
@inlineCallbacks
def test_DBUpgrade(self):
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.old_CalendarUserProxyDatabase(db_path)
+ db = self.old_ProxyDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -113,19 +112,19 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set())
+ db.close()
db = None
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
membersA = yield db.getMembers("A")
membershipsB = yield db.getMemberships("B")
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
@inlineCallbacks
@@ -133,8 +132,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.old_CalendarUserProxyDatabase(db_path)
+ db = self.old_ProxyDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -142,19 +140,19 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set())
+ db.close()
db = None
- db = self.new_CalendarUserProxyDatabase(db_path)
+ db = self.new_ProxyDB(db_path)
membersA = yield db.getMembers("A")
membershipsB = yield db.getMemberships("B")
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
@inlineCallbacks
@@ -162,8 +160,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.new_CalendarUserProxyDatabase(db_path)
+ db = self.new_ProxyDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -171,19 +168,19 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
- db = self.newer_CalendarUserProxyDatabase(db_path)
+ db = self.newer_ProxyDB(db_path)
membersA = yield db.getMembers("A")
membershipsB = yield db.getMemberships("B")
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
@inlineCallbacks
@@ -194,8 +191,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result
yield db.setGroupMembers("A", ("B", "C", "D",))
@@ -235,8 +231,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result
yield db.setGroupMembers("A", ("B", "C", "D",))
@@ -277,8 +272,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result
yield db.setGroupMembers("A", ("B", "C", "D",))
@@ -311,8 +305,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result
yield db.setGroupMembers("A", ("B", "C", "D",))
@@ -353,8 +346,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result for the one we will remove
yield db.setGroupMembers("AA", ("BB", "CC", "DD",))
@@ -375,3 +367,234 @@
self.assertEqual(membershipsDD, set())
self.assertEqual(membershipsEE, set(("AA",)))
+class ProxyPrincipalDBPostgreSQL (twistedcaldav.test.util.TestCase):
+ """
+ Directory service provisioned principals.
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+
+ super(ProxyPrincipalDBPostgreSQL, self).setUp()
+ self.db = ProxyPostgreSQLDB(host="localhost", database="proxies")
+ yield self.db.clean()
+
+ @inlineCallbacks
+ def tearDown(self):
+ yield self.db.close()
+ self.db = None
+
+ @inlineCallbacks
+ def test_normalDB(self):
+
+ # Get the DB
+ yield self.db.clean()
+
+ calendaruserproxy.ProxyDBService = self.db
+ loader = XMLCalendarUserProxyLoader("/Volumes/Data/Users/cyrusdaboo/Documents/Development/Apple/eclipse/CalendarServer-3/conf/auth/proxies-test.xml")
+ yield loader.updateProxyDB()
+
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+
+ membersA = yield self.db.getMembers("A")
+ membershipsB = yield self.db.getMemberships("B")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membershipsB, set(("A",)))
+
+ @inlineCallbacks
+ def test_DBIndexed(self):
+
+ # Get the DB
+ yield self.db.clean()
+ self.assertTrue((yield self.db.queryOne("select hasindexes from pg_tables where tablename = 'groups'")))
+
+ @inlineCallbacks
+ def test_cachingDBInsert(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+
+ membersA = yield self.db.getMembers("A")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+ membershipsE = yield self.db.getMemberships("E")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membershipsB, set(("A",)))
+ self.assertEqual(membershipsC, set(("A",)))
+ self.assertEqual(membershipsD, set(("A",)))
+ self.assertEqual(membershipsE, set(()))
+
+ # Change and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "E",))
+
+ membersA = yield self.db.getMembers("A")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+ membershipsE = yield self.db.getMemberships("E")
+
+ self.assertEqual(membersA, set(("B", "C", "E",)))
+ self.assertEqual(membershipsB, set(("A",)))
+ self.assertEqual(membershipsC, set(("A",)))
+ self.assertEqual(membershipsD, set())
+ self.assertEqual(membershipsE, set(("A",)))
+
+ @inlineCallbacks
+ def test_cachingDBRemove(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+ yield self.db.setGroupMembers("X", ("B", "C",))
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set(("A", "X",)))
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",)))
+
+ # Remove and check the result
+ yield self.db.removeGroup("A")
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set())
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set("X",))
+ self.assertEqual(membershipsC, set("X",))
+ self.assertEqual(membershipsD, set())
+
+ @inlineCallbacks
+ def test_cachingDBRemoveSpecial(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+ yield self.db.setGroupMembers("X", ("B", "C",))
+
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ # Remove and check the result
+ yield self.db.removeGroup("A")
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set())
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set("X",))
+ self.assertEqual(membershipsC, set("X",))
+ self.assertEqual(membershipsD, set())
+
+ @inlineCallbacks
+ def test_cachingDBRemovePrincipal(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+ yield self.db.setGroupMembers("X", ("B", "C",))
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set(("A", "X",)))
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",)))
+
+ # Remove and check the result
+ yield self.db.removePrincipal("B")
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("C", "D",)))
+ self.assertEqual(membersX, set(("C",)))
+ self.assertEqual(membershipsB, set())
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",),))
+
+ @inlineCallbacks
+ def test_cachingDBInsertUncached(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result for the one we will remove
+ yield self.db.setGroupMembers("AA", ("BB", "CC", "DD",))
+ yield self.db.getMemberships("DD")
+
+ # Change and check the result
+ yield self.db.setGroupMembers("AA", ("BB", "CC", "EE",))
+
+ membersAA = yield self.db.getMembers("AA")
+ membershipsBB = yield self.db.getMemberships("BB")
+ membershipsCC = yield self.db.getMemberships("CC")
+ membershipsDD = yield self.db.getMemberships("DD")
+ membershipsEE = yield self.db.getMemberships("EE")
+
+ self.assertEqual(membersAA, set(("BB", "CC", "EE",)))
+ self.assertEqual(membershipsBB, set(("AA",)))
+ self.assertEqual(membershipsCC, set(("AA",)))
+ self.assertEqual(membershipsDD, set())
+ self.assertEqual(membershipsEE, set(("AA",)))
+
+
+try:
+ import pgdb
+except ImportError:
+ ProxyPrincipalDBPostgreSQL.skip = True
+else:
+ try:
+ db = pgdb.connect(host="localhost", database="proxies")
+ except:
+ ProxyPrincipalDBPostgreSQL.skip = True
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -52,8 +52,22 @@
"twistedcaldav.directory.augment.AugmentSqliteDB": {
"dbpath": "/etc/caldavd/augments.sqlite",
},
+ "twistedcaldav.directory.augment.AugmentPostgreSQLDB": {
+ "host": "localhost",
+ "database": "augments",
+ },
}
+DEFAULT_PROXYDB_PARAMS = {
+ "twistedcaldav.directory.calendaruserproxy.ProxySqliteDB": {
+ "dbpath": "/etc/caldavd/proxies.sqlite",
+ },
+ "twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB": {
+ "host": "localhost",
+ "database": "proxies",
+ },
+}
+
DEFAULT_CONFIG = {
# Note: Don't use None values below; that confuses the command-line parser.
@@ -116,11 +130,13 @@
},
#
- # Proxy loader
+ # Proxies
#
- # Allows for initialization of the proxy database from an XML file.
- #
- "ProxyLoadFromFile": "",
+ "ProxyDBService": {
+ "type": "twistedcaldav.directory.calendaruserproxy.ProxySqliteDB",
+ "params": DEFAULT_PROXYDB_PARAMS["twistedcaldav.directory.calendaruserproxy.ProxySqliteDB"],
+ },
+ "ProxyLoadFromFile": "", # Allows for initialization of the proxy database from an XML file
#
# Special principals
@@ -479,6 +495,13 @@
log.warn("Parameter %s is not supported by service %s" % (param, configDict.AugmentService.type))
del configDict.AugmentService.params[param]
+def _postUpdateProxyDBService(configDict):
+ if configDict.ProxyDBService.type in DEFAULT_PROXYDB_PARAMS:
+ for param in tuple(configDict.ProxyDBService.params):
+ if param not in DEFAULT_PROXYDB_PARAMS[configDict.ProxyDBService.type]:
+ log.warn("Parameter %s is not supported by service %s" % (param, configDict.ProxyDBService.type))
+ del configDict.ProxyDBService.params[param]
+
def _updateACLs(configDict):
#
# Base resource ACLs
@@ -687,6 +710,7 @@
_updateHostName,
_postUpdateDirectoryService,
_postUpdateAugmentService,
+ _postUpdateProxyDBService,
_updateACLs,
_updateRejectClients,
_updateDropBox,
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/upgrade.py 2009-09-28 19:22:22 UTC (rev 4557)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/upgrade.py 2009-09-28 20:37:17 UTC (rev 4558)
@@ -19,7 +19,6 @@
from twisted.web2.dav.fileop import rmdir
from twisted.web2.dav import davxml
from twistedcaldav.directory.directory import DirectoryService
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
from twistedcaldav.mail import MailGatewayTokensDatabase
from twistedcaldav.log import Logger
@@ -199,44 +198,44 @@
def doProxyDatabaseMoveUpgrade(config, uid=-1, gid=-1):
+ pass
+# # See if the new one is already present
+# newDbPath = os.path.join(config.DataRoot,
+# CalendarUserProxyDatabase.dbFilename)
+# if os.path.exists(newDbPath):
+# # Nothing to be done, it's already in the new location
+# return
+#
+# # See if the old DB is present
+# oldDbPath = os.path.join(config.DocumentRoot, "principals",
+# CalendarUserProxyDatabase.dbOldFilename)
+# if not os.path.exists(oldDbPath):
+# # Nothing to be moved
+# return
+#
+# # Now move the old one to the new location
+# try:
+# if not os.path.exists(config.DataRoot):
+# makeDirsUserGroup(config.DataRoot, uid=uid, gid=gid)
+# try:
+# os.rename(oldDbPath, newDbPath)
+# except OSError:
+# # Can't rename, must copy/delete
+# shutil.copy2(oldDbPath, newDbPath)
+# os.remove(oldDbPath)
+#
+# except Exception, e:
+# raise UpgradeError(
+# "Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' due to %s."
+# % (oldDbPath, newDbPath, str(e))
+# )
+#
+# log.debug(
+# "Moved the calendar user proxy database from '%s' to '%s'."
+# % (oldDbPath, newDbPath,)
+# )
- # See if the new one is already present
- newDbPath = os.path.join(config.DataRoot,
- CalendarUserProxyDatabase.dbFilename)
- if os.path.exists(newDbPath):
- # Nothing to be done, it's already in the new location
- return
- # See if the old DB is present
- oldDbPath = os.path.join(config.DocumentRoot, "principals",
- CalendarUserProxyDatabase.dbOldFilename)
- if not os.path.exists(oldDbPath):
- # Nothing to be moved
- return
-
- # Now move the old one to the new location
- try:
- if not os.path.exists(config.DataRoot):
- makeDirsUserGroup(config.DataRoot, uid=uid, gid=gid)
- try:
- os.rename(oldDbPath, newDbPath)
- except OSError:
- # Can't rename, must copy/delete
- shutil.copy2(oldDbPath, newDbPath)
- os.remove(oldDbPath)
-
- except Exception, e:
- raise UpgradeError(
- "Upgrade Error: unable to move the old calendar user proxy database at '%s' to '%s' due to %s."
- % (oldDbPath, newDbPath, str(e))
- )
-
- log.debug(
- "Moved the calendar user proxy database from '%s' to '%s'."
- % (oldDbPath, newDbPath,)
- )
-
-
def moveCalendarHome(oldHome, newHome, uid=-1, gid=-1):
if os.path.exists(newHome):
# Both old and new homes exist; stop immediately to let the
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090928/7a44d899/attachment-0001.html>
More information about the calendarserver-changes
mailing list