[CalendarServer-changes] [4561] CalendarServer/branches/users/cdaboo/deployment-partition-4524
source_changes at macosforge.org
source_changes at macosforge.org
Tue Sep 29 11:03:38 PDT 2009
Revision: 4561
http://trac.macosforge.org/projects/calendarserver/changeset/4561
Author: cdaboo at apple.com
Date: 2009-09-29 11:03:38 -0700 (Tue, 29 Sep 2009)
Log Message:
-----------
Backport proxyDB & PostgreSQL changes to deployment.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd-test.plist
CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd.plist
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/database.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/augment.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxyloader.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_augment.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_principal.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipaldb.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py
CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/test/test_database.py
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd-test.plist 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd-test.plist 2009-09-29 18:03:38 UTC (rev 4561)
@@ -161,6 +161,53 @@
</dict>
-->
+ <!-- PostgreSQL Augment Service -->
+ <!--
+ <key>AugmentService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.augment.AugmentPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>augments</string>
+ </dict>
+ </dict>
+ -->
+
+ <!-- Sqlite ProxyDB Service -->
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxySqliteDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>dbpath</key>
+ <string>data/proxies.sqlite</string>
+ </dict>
+ </dict>
+
+ <!-- PostgreSQL ProxyDB Service -->
+ <!--
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>proxies</string>
+ </dict>
+ </dict>
+ -->
+
<key>ProxyLoadFromFile</key>
<string>conf/auth/proxies-test.xml</string>
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd.plist 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/conf/caldavd.plist 2009-09-29 18:03:38 UTC (rev 4561)
@@ -164,7 +164,54 @@
</dict>
-->
+ <!-- PostgreSQL Augment Service -->
<!--
+ <key>AugmentService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.augment.AugmentPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>augments</string>
+ </dict>
+ </dict>
+ -->
+
+ <!-- Sqlite ProxyDB Service -->
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxySqliteDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>dbpath</key>
+ <string>/etc/caldavd/proxies.sqlite</string>
+ </dict>
+ </dict>
+
+ <!-- PostgreSQL ProxyDB Service -->
+ <!--
+ <key>ProxyDBService</key>
+ <dict>
+ <key>type</key>
+ <string>twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB</string>
+
+ <key>params</key>
+ <dict>
+ <key>host</key>
+ <string>localhost</string>
+ <key>database</key>
+ <string>proxies</string>
+ </dict>
+ </dict>
+ -->
+
+ <!--
Special principals
These principals are granted special access and/or perform
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -82,9 +82,23 @@
"twistedcaldav.directory.augment.AugmentSqliteDB": {
"dbpath": "/etc/caldavd/augments.sqlite",
},
+ "twistedcaldav.directory.augment.AugmentPostgreSQLDB": {
+ "host": "localhost",
+ "database": "augments",
+ },
}
+proxyDBDefaultParams = {
+ "twistedcaldav.directory.calendaruserproxy.ProxySqliteDB": {
+ "dbpath": "/etc/caldavd/proxies.sqlite",
+ },
+ "twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB": {
+ "host": "localhost",
+ "database": "proxies",
+ },
+}
+
defaultConfig = {
# Note: Don't use None values below; that confuses the command-line parser.
@@ -140,11 +154,13 @@
},
#
- # Proxy loader
+ # Proxies
#
- # Allows for initialization of the proxy database from an XML file.
- #
- "ProxyLoadFromFile": "",
+ "ProxyDBService": {
+ "type": "twistedcaldav.directory.calendaruserproxy.ProxySqliteDB",
+ "params": proxyDBDefaultParams["twistedcaldav.directory.calendaruserproxy.ProxySqliteDB"],
+ },
+ "ProxyLoadFromFile": "", # Allows for initialization of the proxy database from an XML file
#
# Special principals
@@ -475,6 +491,12 @@
log.warn("Parameter %s is not supported by service %s" % (param, self._data.AugmentService.type))
del self._data.AugmentService.params[param]
+ if self._data.ProxyDBService.type in proxyDBDefaultParams:
+ for param in tuple(self._data.ProxyDBService.params):
+ if param not in proxyDBDefaultParams[self._data.ProxyDBService.type]:
+ log.warn("Parameter %s is not supported by service %s" % (param, self._data.ProxyDBService.type))
+ del self._data.ProxyDBService.params[param]
+
@staticmethod
def updateACLs(self, items):
#
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/database.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/database.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/database.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -14,14 +14,20 @@
# limitations under the License.
##
-from twistedcaldav.log import Logger
-
from twisted.enterprise.adbapi import ConnectionPool
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.python.threadpool import ThreadPool
+from twistedcaldav.config import ConfigurationError
+from twistedcaldav.log import Logger
+
import thread
+try:
+ import pgdb
+except:
+ pgdb = None
+
"""
Generic ADAPI database access object.
"""
@@ -133,6 +139,14 @@
self.initialized = False
@inlineCallbacks
+ def clean(self):
+
+ if not self.initialized:
+ yield self.open()
+
+ yield self._db_empty_data_tables()
+
+ @inlineCallbacks
def execute(self, sql, *query_params):
if not self.initialized:
@@ -187,13 +201,8 @@
"""
raise NotImplementedError
- @inlineCallbacks
def _test_schema_table(self):
- result = (yield self._db_value_for_sql("""
- select (1) from SQLITE_MASTER
- where TYPE = 'table' and NAME = 'CALDAV'
- """))
- returnValue(result)
+ return self._test_table("CALDAV")
@inlineCallbacks
def _db_init(self):
@@ -221,16 +230,13 @@
#
# CALDAV table keeps track of our schema version and type
#
+ yield self._create_table("CALDAV", (
+ ("KEY", "text unique"),
+ ("VALUE", "text unique"),
+ ), True)
+
yield self._db_execute(
"""
- create table if not exists CALDAV (
- KEY text unique,
- VALUE text unique
- )
- """
- )
- yield self._db_execute(
- """
insert or ignore into CALDAV (KEY, VALUE)
values ('SCHEMA_VERSION', :1)
""", (self._db_version(),)
@@ -248,6 +254,14 @@
"""
raise NotImplementedError
+ def _db_empty_data_tables(self):
+ """
+ Delete the database tables.
+ """
+
+ # Implementations can override this to re-create data
+ pass
+
def _db_recreate(self):
"""
Recreate the database tables.
@@ -318,6 +332,7 @@
@raise AssertionError: if the query yields multiple columns.
"""
+ sql = self._prepare_statement(sql)
results = (yield self.pool.runQuery(sql, *query_params))
returnValue(tuple(results))
@@ -333,6 +348,7 @@
@raise AssertionError: if the query yields multiple columns.
"""
+ sql = self._prepare_statement(sql)
results = (yield self.pool.runQuery(sql, *query_params))
returnValue(tuple([row[0] for row in results]))
@@ -363,4 +379,178 @@
C{sql} with C{query_params}.
"""
+ sql = self._prepare_statement(sql)
return self.pool.runOperation(sql, *query_params)
+
+ """
+ Since different databases support different types of columns and modifiers on those we need to
+ have an "abstract" way of specifying columns in our code and then map the abstract specifiers to
+ the underlying DB's allowed types.
+
+ Types we can use are:
+
+ integer
+ text
+ text(n)
+ date
+ serial
+
+ The " unique" modifier can be appended to any of those.
+ """
+ def _map_column_types(self, type):
+ raise NotImplementedError
+
+ def _create_table(self, name, columns, ifnotexists=False):
+ raise NotImplementedError
+
+ def _test_table(self, name):
+ raise NotImplementedError
+
+ def _prepare_statement(self, sql):
+ raise NotImplementedError
+
+class ADBAPISqliteMixin(object):
+
+ @classmethod
+ def _map_column_types(self, coltype):
+
+ result = ""
+ splits = coltype.split()
+ if splits[0] == "integer":
+ result = "integer"
+ elif splits[0] == "text":
+ result = "text"
+ elif splits[0].startswith("text("):
+ result = splits[0]
+ elif splits[0] == "date":
+ result = "date"
+ elif splits[0] == "serial":
+ result = "integer primary key autoincrement"
+
+ if len(splits) > 1 and splits[1] == "unique":
+ result += " unique"
+
+ return result
+
+ @inlineCallbacks
+ def _create_table(self, name, columns, ifnotexists=False):
+
+ colDefs = ["%s %s" % (colname, self._map_column_types(coltype)) for colname, coltype in columns]
+ statement = "create table %s%s (%s)" % (
+ "if not exists " if ifnotexists else "",
+ name,
+ ", ".join(colDefs),
+ )
+ yield self._db_execute(statement)
+
+ @inlineCallbacks
+ def _test_table(self, name):
+ result = (yield self._db_value_for_sql("""
+ select (1) from SQLITE_MASTER
+ where TYPE = 'table' and NAME = '%s'
+ """ % (name,)))
+ returnValue(result)
+
+ def _prepare_statement(self, sql):
+ # We are going to use the sqlite syntax of :1, :2 etc for our
+ # internal statements so we do not need to remap those
+ return sql
+
+if pgdb:
+
+ class ADBAPIPostgreSQLMixin(object):
+
+ @classmethod
+ def _map_column_types(self, coltype):
+
+ result = ""
+ splits = coltype.split()
+ if splits[0] == "integer":
+ result = "integer"
+ elif splits[0] == "text":
+ result = "text"
+ elif splits[0].startswith("text("):
+ result = "char" + splits[0][4:]
+ elif splits[0] == "date":
+ result = "date"
+ elif splits[0] == "serial":
+ result = "serial"
+
+ if len(splits) > 1 and splits[1] == "unique":
+ result += " unique"
+
+ return result
+
+ @inlineCallbacks
+ def _create_table(self, name, columns, ifnotexists=False):
+
+ colDefs = ["%s %s" % (colname, self._map_column_types(coltype)) for colname, coltype in columns]
+ statement = "create table %s (%s)" % (
+ name,
+ ", ".join(colDefs),
+ )
+
+ try:
+ yield self._db_execute(statement)
+ except pgdb.DatabaseError:
+
+ if not ifnotexists:
+ raise
+
+ result = (yield self._test_table(name))
+ if not result:
+ raise
+
+ @inlineCallbacks
+ def _test_table(self, name):
+ result = (yield self._db_value_for_sql("""
+ select * from pg_tables
+ where tablename = '%s'
+ """ % (name.lower(),)))
+ returnValue(result)
+
+ @inlineCallbacks
+ def _db_init_schema_table(self):
+ """
+ Initialise the underlying database tables.
+ @param db_filename: the file name of the index database.
+ @param q: a database cursor to use.
+ """
+
+ #
+ # CALDAV table keeps track of our schema version and type
+ #
+ try:
+ yield self._create_table("CALDAV", (
+ ("KEY", "text unique"),
+ ("VALUE", "text unique"),
+ ), True)
+
+ yield self._db_execute(
+ """
+ insert into CALDAV (KEY, VALUE)
+ values ('SCHEMA_VERSION', :1)
+ """, (self._db_version(),)
+ )
+ yield self._db_execute(
+ """
+ insert into CALDAV (KEY, VALUE)
+ values ('TYPE', :1)
+ """, (self._db_type(),)
+ )
+ except pgdb.DatabaseError:
+ pass
+
+ def _prepare_statement(self, sql):
+ # Convert :1, :2 etc format into %s
+ ctr = 1
+ while sql.find(":%d" % (ctr,)) != -1:
+ sql = sql.replace(":%d" % (ctr,), "%s")
+ ctr += 1
+ return sql
+
+else:
+ class ADBAPIPostgreSQLMixin(object):
+
+ def __init__(self):
+ raise ConfigurationError("PostgreSQL module not available.")
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/augment.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/augment.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/augment.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -16,7 +16,8 @@
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
-from twistedcaldav.database import AbstractADBAPIDatabase
+from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\
+ ADBAPIPostgreSQLMixin
from twistedcaldav.directory.xmlaugmentsparser import XMLAugmentsParser
import time
@@ -243,43 +244,48 @@
@inlineCallbacks
def _db_init_data_tables(self):
"""
- Initialise the underlying database tables.
+ Initialize the underlying database tables.
"""
#
# TESTTYPE table
#
- yield self._db_execute(
- """
- create table AUGMENTS (
- GUID text unique,
- ENABLED text(1),
- PARTITIONID text,
- CALENDARING text(1),
- AUTOSCHEDULE text(1),
- CUADDRS text
- )
- """
- )
- yield self._db_execute(
- """
- create table PARTITIONS (
- PARTITIONID integer primary key autoincrement,
- HOSTEDAT text
- )
- """
- )
+ yield self._create_table("AUGMENTS", (
+ ("GUID", "text unique"),
+ ("ENABLED", "text(1)"),
+ ("PARTITIONID", "text"),
+ ("CALENDARING", "text(1)"),
+ ("AUTOSCHEDULE", "text(1)"),
+ ("CUADDRS", "text"),
+ ))
+ yield self._create_table("PARTITIONS", (
+ ("PARTITIONID", "serial"),
+ ("HOSTEDAT", "text"),
+ ))
+
@inlineCallbacks
- def _db_remove_data_tables(self):
- yield self._db_execute("drop table if exists AUGMENTS")
- yield self._db_execute("drop table if exists PARTITIONS")
+ def _db_empty_data_tables(self):
+ yield self._db_execute("delete from AUGMENTS")
+ yield self._db_execute("delete from PARTITIONS")
-class AugmentSqliteDB(AugmentADAPI):
+class AugmentSqliteDB(ADBAPISqliteMixin, AugmentADAPI):
"""
Sqlite based augment database implementation.
"""
def __init__(self, dbpath):
- super(AugmentSqliteDB, self).__init__("Augments", "sqlite3", (dbpath,))
+ ADBAPISqliteMixin.__init__(self)
+ AugmentADAPI.__init__(self, "Augments", "sqlite3", (dbpath,))
+
+class AugmentPostgreSQLDB(ADBAPIPostgreSQLMixin, AugmentADAPI):
+ """
+ PostgreSQL based augment database implementation.
+ """
+
+ def __init__(self, host, database):
+
+ ADBAPIPostgreSQLMixin.__init__(self)
+ AugmentADAPI.__init__(self, "Augments", "pgdb", (), host=host, database=database,)
+
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -20,6 +20,10 @@
__all__ = [
"CalendarUserProxyPrincipalResource",
+ "ProxyDB",
+ "ProxyDBService",
+ "ProxySqliteDB",
+ "ProxyPostgreSQLDB",
]
from twisted.internet.defer import returnValue
@@ -32,16 +36,16 @@
from twisted.web2.dav.noneprops import NonePropertyStore
from twistedcaldav.config import config
+from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\
+ ADBAPIPostgreSQLMixin
from twistedcaldav.extensions import DAVFile, DAVPrincipalResource
from twistedcaldav.extensions import ReadOnlyWritePropertiesResourceMixIn
from twistedcaldav.memcacher import Memcacher
from twistedcaldav.resource import CalDAVComplianceMixIn
from twistedcaldav.directory.util import NotFilePath
-from twistedcaldav.sql import AbstractSQLDatabase
from twistedcaldav.log import LoggingMixIn
import itertools
-import os
import time
class PermissionsMixIn (ReadOnlyWritePropertiesResourceMixIn):
@@ -61,12 +65,14 @@
)
# Add admins
- aces += tuple([davxml.ACE(
- davxml.Principal(davxml.HRef(principal)),
- davxml.Grant(davxml.Privilege(davxml.All())),
- davxml.Protected(),
- ) for principal in config.AdminPrincipals
- ])
+ aces += tuple((
+ davxml.ACE(
+ davxml.Principal(davxml.HRef(principal)),
+ davxml.Grant(davxml.Privilege(davxml.All())),
+ davxml.Protected(),
+ )
+ for principal in config.AdminPrincipals
+ ))
return davxml.ACL(*aces)
@@ -118,14 +124,10 @@
"""
Return the SQL database for this group principal.
- @return: the L{CalendarUserProxyDatabase} for the principal collection.
+ @return: the L{ProxyDB} for the principal collection.
"""
+ return ProxyDBService
- # The db is located in the principal collection root
- if not hasattr(self.pcollection, "calendar_user_proxy_db"):
- setattr(self.pcollection, "calendar_user_proxy_db", CalendarUserProxyDatabase(config.DataRoot))
- return self.pcollection.calendar_user_proxy_db
-
def resourceType(self):
if self.proxyType == "calendar-proxy-read":
return davxml.ResourceType.calendarproxyread
@@ -336,8 +338,7 @@
for uid in missing:
cacheTimeout = config.DirectoryService.params.get("cacheTimeout", 30) * 60 # in seconds
- yield self._index().removePrincipal(uid,
- delay=cacheTimeout*2)
+ yield self._index().removePrincipal(uid, delay=cacheTimeout*2)
returnValue(found)
@@ -350,7 +351,7 @@
memberships = yield self._index().getMemberships(self.uid)
returnValue([p for p in [self.pcollection.principalForUID(uid) for uid in memberships] if p])
-class CalendarUserProxyDatabase(AbstractSQLDatabase, LoggingMixIn):
+class ProxyDB(AbstractADBAPIDatabase, LoggingMixIn):
"""
A database to maintain calendar user proxy group memberships.
@@ -362,17 +363,16 @@
"""
- dbType = "CALENDARUSERPROXY"
- dbFilename = "calendaruserproxy.sqlite"
- dbFormatVersion = "4"
-
+ schema_version = "4"
+ schema_type = "ProxyDB"
+
class ProxyDBMemcacher(Memcacher):
def setMembers(self, guid, members):
- return self.set("members:%s" % (guid,), str(",".join(members)))
+ return self.set("members:%s" % (str(guid),), str(",".join(members)))
def setMemberships(self, guid, memberships):
- return self.set("memberships:%s" % (guid,), str(",".join(memberships)))
+ return self.set("memberships:%s" % (str(guid),), str(",".join(memberships)))
def getMembers(self, guid):
def _value(value):
@@ -382,7 +382,7 @@
return None
else:
return set()
- d = self.get("members:%s" % (guid,))
+ d = self.get("members:%s" % (str(guid),))
d.addCallback(_value)
return d
@@ -394,15 +394,15 @@
return None
else:
return set()
- d = self.get("memberships:%s" % (guid,))
+ d = self.get("memberships:%s" % (str(guid),))
d.addCallback(_value)
return d
def deleteMember(self, guid):
- return self.delete("members:%s" % (guid,))
+ return self.delete("members:%s" % (str(guid),))
def deleteMembership(self, guid):
- return self.delete("memberships:%s" % (guid,))
+ return self.delete("memberships:%s" % (str(guid),))
def setDeletionTimer(self, guid, delay):
return self.set("del:%s" % (str(guid),), str(self.getTime()+delay))
@@ -431,11 +431,10 @@
theTime = int(time.time())
return theTime
- def __init__(self, path):
- path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
- super(CalendarUserProxyDatabase, self).__init__(path, True)
+ def __init__(self, dbID, dbapiName, dbapiArgs, **kwargs):
+ AbstractADBAPIDatabase.__init__(self, dbID, dbapiName, dbapiArgs, True, **kwargs)
- self._memcacher = CalendarUserProxyDatabase.ProxyDBMemcacher("proxyDB")
+ self._memcacher = ProxyDB.ProxyDBMemcacher("proxyDB")
@inlineCallbacks
def setGroupMembers(self, principalUID, members):
@@ -452,17 +451,19 @@
current_members = ()
current_members = set(current_members)
- self.setGroupMembersInDatabase(principalUID, members)
-
- # Update cache
+ # Find changes
update_members = set(members)
-
remove_members = current_members.difference(update_members)
add_members = update_members.difference(current_members)
+
+ yield self.changeGroupMembersInDatabase(principalUID, add_members, remove_members)
+
+ # Update cache
for member in itertools.chain(remove_members, add_members,):
_ignore = yield self._memcacher.deleteMembership(member)
_ignore = yield self._memcacher.deleteMember(principalUID)
+ @inlineCallbacks
def setGroupMembersInDatabase(self, principalUID, members):
"""
A blocking call to add a group membership record in the database.
@@ -471,11 +472,25 @@
@param members: a list UIDs of principals that are members of this group.
"""
# Remove what is there, then add it back.
- self._delete_from_db(principalUID)
- self._add_to_db(principalUID, members)
- self._db_commit()
+ yield self._delete_from_db(principalUID)
+ yield self._add_to_db(principalUID, members)
@inlineCallbacks
+ def changeGroupMembersInDatabase(self, principalUID, addMembers, removeMembers):
+ """
+ A blocking call to add a group membership record in the database.
+
+ @param principalUID: the UID of the group principal to add.
+ @param addMembers: a list UIDs of principals to be added as members of this group.
+ @param removeMembers: a list UIDs of principals to be removed as members of this group.
+ """
+ # Remove what is there, then add it back.
+ for member in removeMembers:
+ yield self._delete_from_db_one(principalUID, member)
+ for member in addMembers:
+ yield self._add_to_db_one(principalUID, member)
+
+ @inlineCallbacks
def removeGroup(self, principalUID):
"""
Remove a group membership record.
@@ -486,8 +501,7 @@
# Need to get the members before we do the delete
members = yield self.getMembers(principalUID)
- self._delete_from_db(principalUID)
- self._db_commit()
+ yield self._delete_from_db(principalUID)
# Update cache
if members:
@@ -521,7 +535,7 @@
# No timer was previously set
self.log_debug("Delaying removal of missing proxy principal '%s'" %
(principalUID,))
- self._memcacher.setDeletionTimer(principalUID, delay=delay)
+ yield self._memcacher.setDeletionTimer(principalUID, delay=delay)
returnValue(None)
self.log_warn("Removing missing proxy principal for '%s'" %
@@ -529,7 +543,7 @@
for suffix in ("calendar-proxy-read", "calendar-proxy-write",):
groupUID = "%s#%s" % (principalUID, suffix,)
- self._delete_from_db(groupUID)
+ yield self._delete_from_db(groupUID)
# Update cache
members = yield self.getMembers(groupUID)
@@ -542,29 +556,27 @@
for groupUID in memberships:
yield self._memcacher.deleteMember(groupUID)
- self._delete_from_db_member(principalUID)
+ yield self._delete_from_db_member(principalUID)
yield self._memcacher.deleteMembership(principalUID)
- self._db_commit()
- self._memcacher.clearDeletionTimer(principalUID)
+ yield self._memcacher.clearDeletionTimer(principalUID)
@inlineCallbacks
def getMembers(self, principalUID):
"""
Return the list of group member UIDs for the specified principal.
-
+
@return: a deferred returning a C{set} of members.
"""
+ @inlineCallbacks
def _members():
- members = set()
- for row in self._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", principalUID):
- members.add(row[0])
- return members
+ result = set([row[0] for row in (yield self.query("select MEMBER from GROUPS where GROUPNAME = :1", (principalUID,)))])
+ returnValue(result)
# Pull from cache
result = yield self._memcacher.getMembers(principalUID)
if result is None:
- result = _members()
+ result = (yield _members())
yield self._memcacher.setMembers(principalUID, result)
returnValue(result)
@@ -576,19 +588,19 @@
@return: a deferred returning a C{set} of memberships.
"""
+ @inlineCallbacks
def _members():
- members = set()
- for row in self._db_execute("select GROUPNAME from GROUPS where MEMBER = :1", principalUID):
- members.add(row[0])
- return members
+ result = set([row[0] for row in (yield self.query("select GROUPNAME from GROUPS where MEMBER = :1", (principalUID,)))])
+ returnValue(result)
# Pull from cache
result = yield self._memcacher.getMemberships(principalUID)
if result is None:
- result = _members()
+ result = (yield _members())
yield self._memcacher.setMemberships(principalUID, result)
returnValue(result)
+ @inlineCallbacks
def _add_to_db(self, principalUID, members):
"""
Insert the specified entry into the database.
@@ -597,42 +609,66 @@
@param members: a list of UIDs or principals that are members of this group.
"""
for member in members:
- self._db_execute(
+ yield self.execute(
"""
insert into GROUPS (GROUPNAME, MEMBER)
values (:1, :2)
- """, principalUID, member
+ """, (principalUID, member,)
)
+ def _add_to_db_one(self, principalUID, memberUID):
+ """
+ Insert the specified entry into the database.
+
+ @param principalUID: the UID of the group principal to add.
+ @param memberUID: the UID of the principal that is being added as a member of this group.
+ """
+ return self.execute(
+ """
+ insert into GROUPS (GROUPNAME, MEMBER)
+ values (:1, :2)
+ """, (principalUID, memberUID,)
+ )
+
def _delete_from_db(self, principalUID):
"""
Deletes the specified entry from the database.
@param principalUID: the UID of the group principal to remove.
"""
- self._db_execute("delete from GROUPS where GROUPNAME = :1", principalUID)
+ return self.execute("delete from GROUPS where GROUPNAME = :1", (principalUID,))
+ def _delete_from_db_one(self, principalUID, memberUID):
+ """
+ Deletes the specified entry from the database.
+
+ @param principalUID: the UID of the group principal to remove.
+ @param memberUID: the UID of the principal that is being removed as a member of this group.
+ """
+ return self.execute("delete from GROUPS where GROUPNAME = :1 and MEMBER = :2", (principalUID, memberUID,))
+
def _delete_from_db_member(self, principalUID):
"""
Deletes the specified member entry from the database.
@param principalUID: the UID of the member principal to remove.
"""
- self._db_execute("delete from GROUPS where MEMBER = :1", principalUID)
+ return self.execute("delete from GROUPS where MEMBER = :1", (principalUID,))
def _db_version(self):
"""
@return: the schema version assigned to this index.
"""
- return CalendarUserProxyDatabase.dbFormatVersion
+ return ProxyDB.schema_version
def _db_type(self):
"""
@return: the collection type assigned to this index.
"""
- return CalendarUserProxyDatabase.dbType
+ return ProxyDB.schema_type
- def _db_init_data_tables(self, q):
+ @inlineCallbacks
+ def _db_init_data_tables(self):
"""
Initialise the underlying database tables.
@param q: a database cursor to use.
@@ -641,46 +677,90 @@
#
# GROUPS table
#
- q.execute(
+ yield self._create_table("GROUPS", (
+ ("GROUPNAME", "text"),
+ ("MEMBER", "text"),
+ ))
+
+ yield self._db_execute(
"""
- create table GROUPS (
- GROUPNAME text,
- MEMBER text
- )
- """
- )
- q.execute(
- """
create index GROUPNAMES on GROUPS (GROUPNAME)
"""
)
- q.execute(
+ yield self._db_execute(
"""
create index MEMBERS on GROUPS (MEMBER)
"""
)
- def _db_upgrade_data_tables(self, q, old_version):
+ @inlineCallbacks
+ def _db_upgrade_data_tables(self, old_version):
"""
Upgrade the data from an older version of the DB.
- @param q: a database cursor to use.
@param old_version: existing DB's version number
@type old_version: str
"""
# Add index if old version is less than "4"
if int(old_version) < 4:
- q.execute(
+ yield self._db_execute(
"""
create index GROUPNAMES on GROUPS (GROUPNAME)
"""
)
- q.execute(
+ yield self._db_execute(
"""
create index MEMBERS on GROUPS (MEMBER)
"""
)
+ def _db_empty_data_tables(self):
+ """
+ Empty the underlying database tables.
+ @param q: a database cursor to use.
+ """
+
+ #
+ # GROUPS table
+ #
+ return self._db_execute("delete from GROUPS")
+
+ @inlineCallbacks
+ def clean(self):
+
+ if not self.initialized:
+ yield self.open()
+
+ for group in [row[0] for row in (yield self.query("select GROUPNAME from GROUPS"))]:
+ self.removeGroup(group)
+
+ yield super(ProxyDB, self).clean()
+
+
+ProxyDBService = None # Global proxyDB service
+
+
+class ProxySqliteDB(ADBAPISqliteMixin, ProxyDB):
+ """
+ Sqlite based proxy database implementation.
+ """
+
+ def __init__(self, dbpath):
+
+ ADBAPISqliteMixin.__init__(self)
+ ProxyDB.__init__(self, "Proxies", "sqlite3", (dbpath,))
+
+class ProxyPostgreSQLDB(ADBAPIPostgreSQLMixin, ProxyDB):
+ """
+ PostgreSQL based augment database implementation.
+ """
+
+ def __init__(self, host, database):
+
+ ADBAPIPostgreSQLMixin.__init__(self, )
+ ProxyDB.__init__(self, "Proxies", "pgdb", (), host=host, database=database,)
+
+
##
# Utilities
##
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxyloader.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxyloader.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxyloader.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -16,8 +16,7 @@
from xml.etree.ElementTree import ElementTree
from xml.parsers.expat import ExpatError
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
-from twistedcaldav.config import config
+from twistedcaldav.directory import calendaruserproxy
from twisted.internet.defer import inlineCallbacks
import types
@@ -129,7 +128,7 @@
@inlineCallbacks
def updateProxyDB(self):
- db = CalendarUserProxyDatabase(config.DataRoot)
+ db = calendaruserproxy.ProxyDBService
for item in self.items:
guid, write_proxies, read_proxies = item
for proxy in write_proxies:
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -45,7 +45,7 @@
from twistedcaldav.config import config
from twistedcaldav.cache import DisabledCacheNotifier, PropfindCacheMixin
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+from twistedcaldav.directory import calendaruserproxy
from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyPrincipalResource
from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.directory.util import NotFilePath
@@ -482,16 +482,11 @@
"""
Return the SQL database for calendar user proxies.
- @return: the L{CalendarUserProxyDatabase} for the principal collection.
+ @return: the L{ProxyDB} for the principal collection.
"""
- # Get the principal collection we are contained in
- pcollection = self.parent.parent
-
# The db is located in the principal collection root
- if not hasattr(pcollection, "calendar_user_proxy_db"):
- setattr(pcollection, "calendar_user_proxy_db", CalendarUserProxyDatabase(config.DataRoot))
- return pcollection.calendar_user_proxy_db
+ return calendaruserproxy.ProxyDBService
def alternateURIs(self):
# FIXME: Add API to IDirectoryRecord for getting a record URI?
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_augment.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_augment.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_augment.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -15,7 +15,8 @@
##
from twistedcaldav.test.util import TestCase
-from twistedcaldav.directory.augment import AugmentXMLDB, AugmentSqliteDB
+from twistedcaldav.directory.augment import AugmentXMLDB, AugmentSqliteDB,\
+ AugmentPostgreSQLDB
from twisted.internet.defer import inlineCallbacks
from twistedcaldav.directory.xmlaugmentsparser import XMLAugmentsParser
import cStringIO
@@ -106,3 +107,30 @@
yield self._checkRecord(db, item)
yield self._checkNoRecord(db, "D11F03A0-97EA-48AF-9A6C-FAC7F3975767")
+
+class AugmentPostgreSQLTests(AugmentTests):
+
+ @inlineCallbacks
+ def test_read(self):
+
+ db = AugmentPostgreSQLDB("localhost", "augments")
+ yield db.clean()
+
+ dbxml = AugmentXMLDB((xmlFile,))
+ for record in dbxml.db.values():
+ yield db.addAugmentRecord(record)
+
+ for item in testRecords:
+ yield self._checkRecord(db, item)
+
+ yield self._checkNoRecord(db, "D11F03A0-97EA-48AF-9A6C-FAC7F3975767")
+
+try:
+ import pgdb
+except ImportError:
+ AugmentPostgreSQLTests.skip = True
+else:
+ try:
+ db = pgdb.connect(host="localhost", database="augments")
+ except:
+ AugmentPostgreSQLTests.skip = True
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_principal.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_principal.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -25,7 +25,7 @@
from twistedcaldav.static import CalendarHomeProvisioningFile
from twistedcaldav.config import config
-from twistedcaldav.directory import augment
+from twistedcaldav.directory import augment, calendaruserproxy
from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.directory.xmlfile import XMLDirectoryService
from twistedcaldav.directory.test.test_xmlfile import xmlFile, augmentsFile
@@ -63,6 +63,7 @@
self.principalRootResources[directory.__class__.__name__] = provisioningResource
augment.AugmentService = augment.AugmentXMLDB(xmlFiles=(augmentsFile.path,))
+ calendaruserproxy.ProxyDBService = calendaruserproxy.ProxySqliteDB(self.mktemp())
def test_hierarchy(self):
"""
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipaldb.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipaldb.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipaldb.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,18 +15,19 @@
##
from twistedcaldav.config import config
-import os
-
from twisted.internet.defer import inlineCallbacks
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
+from twistedcaldav.directory.calendaruserproxy import ProxySqliteDB,\
+ ProxyPostgreSQLDB
import twistedcaldav.test.util
+from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
+from twistedcaldav.directory import calendaruserproxy
-class ProxyPrincipalDB (twistedcaldav.test.util.TestCase):
+class ProxyPrincipalDBSqlite (twistedcaldav.test.util.TestCase):
"""
Directory service provisioned principals.
"""
- class old_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+ class old_ProxyDB(ProxySqliteDB):
def _db_version(self):
"""
@@ -34,7 +35,7 @@
"""
return "3"
- def _db_init_data_tables(self, q):
+ def _db_init_data_tables(self):
"""
Initialise the underlying database tables.
@param q: a database cursor to use.
@@ -43,7 +44,7 @@
#
# GROUPS table
#
- q.execute(
+ return self.execute(
"""
create table GROUPS (
GROUPNAME text,
@@ -52,7 +53,7 @@
"""
)
- class new_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+ class new_ProxyDB(ProxySqliteDB):
def _db_version(self):
"""
@@ -60,7 +61,7 @@
"""
return "11"
- class newer_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+ class newer_ProxyDB(ProxySqliteDB):
def _db_version(self):
"""
@@ -73,8 +74,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -83,29 +83,28 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
+ @inlineCallbacks
def test_DBIndexed(self):
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+ db = ProxySqliteDB(db_path)
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ @inlineCallbacks
def test_OldDB(self):
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.old_CalendarUserProxyDatabase(db_path)
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+ db = self.old_ProxyDB(db_path)
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set())
@inlineCallbacks
def test_DBUpgrade(self):
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.old_CalendarUserProxyDatabase(db_path)
+ db = self.old_ProxyDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -113,19 +112,19 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set())
+ db.close()
db = None
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
membersA = yield db.getMembers("A")
membershipsB = yield db.getMemberships("B")
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
@inlineCallbacks
@@ -133,8 +132,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.old_CalendarUserProxyDatabase(db_path)
+ db = self.old_ProxyDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -142,19 +140,19 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set())
+ db.close()
db = None
- db = self.new_CalendarUserProxyDatabase(db_path)
+ db = self.new_ProxyDB(db_path)
membersA = yield db.getMembers("A")
membershipsB = yield db.getMemberships("B")
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
@inlineCallbacks
@@ -162,8 +160,7 @@
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = self.new_CalendarUserProxyDatabase(db_path)
+ db = self.new_ProxyDB(db_path)
yield db.setGroupMembers("A", ("B", "C", "D",))
membersA = yield db.getMembers("A")
@@ -171,31 +168,30 @@
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
- db = self.newer_CalendarUserProxyDatabase(db_path)
+ db = self.newer_ProxyDB(db_path)
membersA = yield db.getMembers("A")
membershipsB = yield db.getMemberships("B")
self.assertEqual(membersA, set(("B", "C", "D",)))
self.assertEqual(membershipsB, set(("A",)))
- self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
- db._db_close()
+ self.assertEqual(set([row[1] for row in (yield db.query("PRAGMA index_list(GROUPS)"))]), set(("GROUPNAMES", "MEMBERS")))
+ db.close()
db = None
@inlineCallbacks
def test_cachingDBInsert(self):
for processType in ("Single", "Combined",):
- config.processType = processType
+ config.ProcessType = processType
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result
yield db.setGroupMembers("A", ("B", "C", "D",))
@@ -231,12 +227,11 @@
def test_cachingDBRemove(self):
for processType in ("Single", "Combined",):
- config.processType = processType
+ config.ProcessType = processType
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result
yield db.setGroupMembers("A", ("B", "C", "D",))
@@ -270,15 +265,88 @@
self.assertEqual(membershipsD, set())
@inlineCallbacks
+ def test_cachingDBRemoveSpecial(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ db_path = self.mktemp()
+ db = ProxySqliteDB(db_path)
+
+ # Do one insert and check the result
+ yield db.setGroupMembers("A", ("B", "C", "D",))
+ yield db.setGroupMembers("X", ("B", "C",))
+
+ membershipsB = yield db.getMemberships("B")
+ membershipsC = yield db.getMemberships("C")
+ membershipsD = yield db.getMemberships("D")
+
+ # Remove and check the result
+ yield db.removeGroup("A")
+
+ membersA = yield db.getMembers("A")
+ membersX = yield db.getMembers("X")
+ membershipsB = yield db.getMemberships("B")
+ membershipsC = yield db.getMemberships("C")
+ membershipsD = yield db.getMemberships("D")
+
+ self.assertEqual(membersA, set())
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set("X",))
+ self.assertEqual(membershipsC, set("X",))
+ self.assertEqual(membershipsD, set())
+
+ @inlineCallbacks
+ def test_cachingDBRemovePrincipal(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ db_path = self.mktemp()
+ db = ProxySqliteDB(db_path)
+
+ # Do one insert and check the result
+ yield db.setGroupMembers("A", ("B", "C", "D",))
+ yield db.setGroupMembers("X", ("B", "C",))
+
+ membersA = yield db.getMembers("A")
+ membersX = yield db.getMembers("X")
+ membershipsB = yield db.getMemberships("B")
+ membershipsC = yield db.getMemberships("C")
+ membershipsD = yield db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set(("A", "X",)))
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",)))
+
+ # Remove and check the result
+ yield db.removePrincipal("B")
+
+ membersA = yield db.getMembers("A")
+ membersX = yield db.getMembers("X")
+ membershipsB = yield db.getMemberships("B")
+ membershipsC = yield db.getMemberships("C")
+ membershipsD = yield db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("C", "D",)))
+ self.assertEqual(membersX, set(("C",)))
+ self.assertEqual(membershipsB, set())
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",),))
+
+ @inlineCallbacks
def test_cachingDBInsertUncached(self):
for processType in ("Single", "Combined",):
- config.processType = processType
+ config.ProcessType = processType
# Get the DB
db_path = self.mktemp()
- os.mkdir(db_path)
- db = CalendarUserProxyDatabase(db_path)
+ db = ProxySqliteDB(db_path)
# Do one insert and check the result for the one we will remove
yield db.setGroupMembers("AA", ("BB", "CC", "DD",))
@@ -299,3 +367,234 @@
self.assertEqual(membershipsDD, set())
self.assertEqual(membershipsEE, set(("AA",)))
+class ProxyPrincipalDBPostgreSQL (twistedcaldav.test.util.TestCase):
+ """
+ Directory service provisioned principals.
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+
+ super(ProxyPrincipalDBPostgreSQL, self).setUp()
+ self.db = ProxyPostgreSQLDB(host="localhost", database="proxies")
+ yield self.db.clean()
+
+ @inlineCallbacks
+ def tearDown(self):
+ yield self.db.close()
+ self.db = None
+
+ @inlineCallbacks
+ def test_normalDB(self):
+
+ # Get the DB
+ yield self.db.clean()
+
+ calendaruserproxy.ProxyDBService = self.db
+ loader = XMLCalendarUserProxyLoader("/Volumes/Data/Users/cyrusdaboo/Documents/Development/Apple/eclipse/CalendarServer-3/conf/auth/proxies-test.xml")
+ yield loader.updateProxyDB()
+
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+
+ membersA = yield self.db.getMembers("A")
+ membershipsB = yield self.db.getMemberships("B")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membershipsB, set(("A",)))
+
+ @inlineCallbacks
+ def test_DBIndexed(self):
+
+ # Get the DB
+ yield self.db.clean()
+ self.assertTrue((yield self.db.queryOne("select hasindexes from pg_tables where tablename = 'groups'")))
+
+ @inlineCallbacks
+ def test_cachingDBInsert(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+
+ membersA = yield self.db.getMembers("A")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+ membershipsE = yield self.db.getMemberships("E")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membershipsB, set(("A",)))
+ self.assertEqual(membershipsC, set(("A",)))
+ self.assertEqual(membershipsD, set(("A",)))
+ self.assertEqual(membershipsE, set(()))
+
+ # Change and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "E",))
+
+ membersA = yield self.db.getMembers("A")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+ membershipsE = yield self.db.getMemberships("E")
+
+ self.assertEqual(membersA, set(("B", "C", "E",)))
+ self.assertEqual(membershipsB, set(("A",)))
+ self.assertEqual(membershipsC, set(("A",)))
+ self.assertEqual(membershipsD, set())
+ self.assertEqual(membershipsE, set(("A",)))
+
+ @inlineCallbacks
+ def test_cachingDBRemove(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+ yield self.db.setGroupMembers("X", ("B", "C",))
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set(("A", "X",)))
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",)))
+
+ # Remove and check the result
+ yield self.db.removeGroup("A")
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set())
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set("X",))
+ self.assertEqual(membershipsC, set("X",))
+ self.assertEqual(membershipsD, set())
+
+ @inlineCallbacks
+ def test_cachingDBRemoveSpecial(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+ yield self.db.setGroupMembers("X", ("B", "C",))
+
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ # Remove and check the result
+ yield self.db.removeGroup("A")
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set())
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set("X",))
+ self.assertEqual(membershipsC, set("X",))
+ self.assertEqual(membershipsD, set())
+
+ @inlineCallbacks
+ def test_cachingDBRemovePrincipal(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result
+ yield self.db.setGroupMembers("A", ("B", "C", "D",))
+ yield self.db.setGroupMembers("X", ("B", "C",))
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("B", "C", "D",)))
+ self.assertEqual(membersX, set(("B", "C",)))
+ self.assertEqual(membershipsB, set(("A", "X",)))
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",)))
+
+ # Remove and check the result
+ yield self.db.removePrincipal("B")
+
+ membersA = yield self.db.getMembers("A")
+ membersX = yield self.db.getMembers("X")
+ membershipsB = yield self.db.getMemberships("B")
+ membershipsC = yield self.db.getMemberships("C")
+ membershipsD = yield self.db.getMemberships("D")
+
+ self.assertEqual(membersA, set(("C", "D",)))
+ self.assertEqual(membersX, set(("C",)))
+ self.assertEqual(membershipsB, set())
+ self.assertEqual(membershipsC, set(("A", "X",)))
+ self.assertEqual(membershipsD, set(("A",),))
+
+ @inlineCallbacks
+ def test_cachingDBInsertUncached(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ # Get the DB
+ yield self.db.clean()
+
+ # Do one insert and check the result for the one we will remove
+ yield self.db.setGroupMembers("AA", ("BB", "CC", "DD",))
+ yield self.db.getMemberships("DD")
+
+ # Change and check the result
+ yield self.db.setGroupMembers("AA", ("BB", "CC", "EE",))
+
+ membersAA = yield self.db.getMembers("AA")
+ membershipsBB = yield self.db.getMemberships("BB")
+ membershipsCC = yield self.db.getMemberships("CC")
+ membershipsDD = yield self.db.getMemberships("DD")
+ membershipsEE = yield self.db.getMemberships("EE")
+
+ self.assertEqual(membersAA, set(("BB", "CC", "EE",)))
+ self.assertEqual(membershipsBB, set(("AA",)))
+ self.assertEqual(membershipsCC, set(("AA",)))
+ self.assertEqual(membershipsDD, set())
+ self.assertEqual(membershipsEE, set(("AA",)))
+
+
+try:
+ import pgdb
+except ImportError:
+ ProxyPrincipalDBPostgreSQL.skip = True
+else:
+ try:
+ db = pgdb.connect(host="localhost", database="proxies")
+ except:
+ ProxyPrincipalDBPostgreSQL.skip = True
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -28,7 +28,7 @@
import twistedcaldav.test.util
from twistedcaldav.config import config
-from twistedcaldav.directory import augment
+from twistedcaldav.directory import augment, calendaruserproxy
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
import os
@@ -43,6 +43,7 @@
self.directoryService = XMLDirectoryService(xmlFile=xmlFile)
augment.AugmentService = augment.AugmentXMLDB(xmlFiles=(augmentsFile.path,))
+ calendaruserproxy.ProxyDBService = calendaruserproxy.ProxySqliteDB(self.mktemp())
# Set up a principals hierarchy for each service we're testing with
self.principalRootResources = {}
@@ -381,6 +382,7 @@
# Set up the in-memory (non-null) memcacher:
config.ProcessType = "Single"
+ calendaruserproxy.ProxyDBService._memcacher._memcacheProtocol = None
principal = self._getPrincipalByShortName(
DirectoryService.recordType_users, "wsanchez")
db = principal._calendar_user_proxy_index()
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -46,7 +46,7 @@
from twistedcaldav.config import config, parseConfig, defaultConfig, ConfigurationError
from twistedcaldav.root import RootResource
from twistedcaldav.resource import CalDAVResource
-from twistedcaldav.directory import augment
+from twistedcaldav.directory import augment, calendaruserproxy
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
from twistedcaldav.directory.digest import QopDigestCredentialFactory
from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
@@ -464,6 +464,19 @@
raise
#
+ # Setup the ProxyDB Service
+ #
+ proxydbClass = namedClass(config.ProxyDBService.type)
+
+ log.info("Configuring proxydb service of type: %s" % (proxydbClass,))
+
+ try:
+ calendaruserproxy.ProxyDBService = proxydbClass(**config.ProxyDBService.params)
+ except IOError, e:
+ log.error("Could not start proxydb service")
+ raise
+
+ #
# Setup the Directory
#
directories = []
@@ -506,7 +519,7 @@
loader = XMLCalendarUserProxyLoader(config.ProxyLoadFromFile)
return loader.updateProxyDB()
- reactor.addSystemEventTrigger("before", "startup", _doProxyUpdate)
+ reactor.addSystemEventTrigger("after", "startup", _doProxyUpdate)
#
# Configure Memcached Client Pool
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/test/test_database.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/test/test_database.py 2009-09-29 18:02:45 UTC (rev 4560)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/test/test_database.py 2009-09-29 18:03:38 UTC (rev 4561)
@@ -14,7 +14,7 @@
# limitations under the License.
##
-from twistedcaldav.database import AbstractADBAPIDatabase
+from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin
import twistedcaldav.test.util
from twisted.internet.defer import inlineCallbacks
@@ -27,7 +27,7 @@
Test abstract SQL DB class
"""
- class TestDB(AbstractADBAPIDatabase):
+ class TestDB(ADBAPISqliteMixin, AbstractADBAPIDatabase):
def __init__(self, path, persistent=False, version="1"):
self.version = version
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090929/9dfd97fa/attachment-0001.html>
More information about the calendarserver-changes
mailing list