<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[11891] CalendarServer/branches/users/sagen/groupcacher</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/11891">11891</a></dd>
<dt>Author</dt> <dd>sagen@apple.com</dd>
<dt>Date</dt> <dd>2013-11-06 12:11:38 -0800 (Wed, 06 Nov 2013)</dd>
</dl>
<h3>Log Message</h3>
<pre>Checkpoint of group cacher work: group membership and delegates are in the store</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserssagengroupcachercalendarservertapcaldavpy">CalendarServer/branches/users/sagen/groupcacher/calendarserver/tap/caldav.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachercalendarservertoolsprincipalspy">CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/principals.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachercalendarservertoolsutilpy">CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/util.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwextenterprisedalrecordpy">CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/record.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwextenterprisedalsyntaxpy">CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/syntax.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwistedcaldavdirectorydirectorypy">CalendarServer/branches/users/sagen/groupcacher/twistedcaldav/directory/directory.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertxdavcommondatastoresqlpy">CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertxdavcommondatastoresql_schemacurrentsql">CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql_schema/current.sql</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwextwhodelegatespy">CalendarServer/branches/users/sagen/groupcacher/twext/who/delegates.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwextwhogroupspy">CalendarServer/branches/users/sagen/groupcacher/twext/who/groups.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwextwhotesttest_delegatespy">CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_delegates.py</a></li>
<li><a href="#CalendarServerbranchesuserssagengroupcachertwextwhotesttest_groupspy">CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_groups.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserssagengroupcachercalendarservertapcaldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/calendarserver/tap/caldav.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/calendarserver/tap/caldav.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/calendarserver/tap/caldav.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -81,11 +81,11 @@
</span><span class="cx"> from twistedcaldav.config import config, ConfigurationError
</span><span class="cx"> from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
</span><span class="cx"> from twistedcaldav.directory import calendaruserproxy
</span><del>-from twistedcaldav.directory.directory import GroupMembershipCacheUpdater
</del><ins>+from twext.who.groups import GroupCacher
</ins><span class="cx"> from twistedcaldav.localization import processLocalizationFiles
</span><span class="cx"> from twistedcaldav import memcachepool
</span><span class="cx"> from twistedcaldav.upgrade import UpgradeFileSystemFormatStep, PostDBImportStep
</span><del>-from twistedcaldav.directory.directory import scheduleNextGroupCachingUpdate
</del><ins>+from twext.who.groups import scheduleNextGroupCachingUpdate
</ins><span class="cx">
</span><span class="cx"> try:
</span><span class="cx"> from twistedcaldav.authkerb import NegotiateCredentialFactory
</span><span class="lines">@@ -890,9 +890,10 @@
</span><span class="cx">
</span><span class="cx"> # Optionally set up group cacher
</span><span class="cx"> if config.GroupCaching.Enabled:
</span><del>- groupCacher = GroupMembershipCacheUpdater(
</del><ins>+ groupCacher = GroupCacher(
</ins><span class="cx"> calendaruserproxy.ProxyDBService,
</span><span class="cx"> directory,
</span><ins>+ store,
</ins><span class="cx"> config.GroupCaching.UpdateSeconds,
</span><span class="cx"> config.GroupCaching.ExpireSeconds,
</span><span class="cx"> config.GroupCaching.LockSeconds,
</span><span class="lines">@@ -1200,9 +1201,10 @@
</span><span class="cx">
</span><span class="cx"> # Optionally set up group cacher
</span><span class="cx"> if config.GroupCaching.Enabled:
</span><del>- groupCacher = GroupMembershipCacheUpdater(
</del><ins>+ groupCacher = GroupCacher(
</ins><span class="cx"> calendaruserproxy.ProxyDBService,
</span><span class="cx"> directory,
</span><ins>+ store,
</ins><span class="cx"> config.GroupCaching.UpdateSeconds,
</span><span class="cx"> config.GroupCaching.ExpireSeconds,
</span><span class="cx"> config.GroupCaching.LockSeconds,
</span><span class="lines">@@ -1713,9 +1715,10 @@
</span><span class="cx">
</span><span class="cx"> # Optionally set up group cacher
</span><span class="cx"> if config.GroupCaching.Enabled:
</span><del>- groupCacher = GroupMembershipCacheUpdater(
</del><ins>+ groupCacher = GroupCacher(
</ins><span class="cx"> calendaruserproxy.ProxyDBService,
</span><span class="cx"> directory,
</span><ins>+ store,
</ins><span class="cx"> config.GroupCaching.UpdateSeconds,
</span><span class="cx"> config.GroupCaching.ExpireSeconds,
</span><span class="cx"> config.GroupCaching.LockSeconds,
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachercalendarservertoolsprincipalspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/principals.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/principals.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/principals.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -31,7 +31,7 @@
</span><span class="cx">
</span><span class="cx"> from twistedcaldav.config import config
</span><span class="cx"> from twistedcaldav.directory.directory import UnknownRecordTypeError, DirectoryError
</span><del>-from twistedcaldav.directory.directory import schedulePolledGroupCachingUpdate
</del><ins>+from twext.who.groups import schedulePolledGroupCachingUpdate
</ins><span class="cx">
</span><span class="cx"> from calendarserver.tools.util import (
</span><span class="cx"> booleanArgument, proxySubprincipal, action_addProxyPrincipal,
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachercalendarservertoolsutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/util.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/util.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/calendarserver/tools/util.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -50,7 +50,7 @@
</span><span class="cx"> from twistedcaldav.directory import calendaruserproxy
</span><span class="cx"> from twistedcaldav.directory.aggregate import AggregateDirectoryService
</span><span class="cx"> from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
</span><del>-from twistedcaldav.directory.directory import schedulePolledGroupCachingUpdate
</del><ins>+from twext.who.groups import schedulePolledGroupCachingUpdate
</ins><span class="cx"> from calendarserver.push.notifier import NotifierFactory
</span><span class="cx">
</span><span class="cx"> from txdav.common.datastore.file import CommonDataStore
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwextenterprisedalrecordpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/record.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/record.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/record.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -163,7 +163,7 @@
</span><span class="cx"> """
</span><span class="cx"> words = columnName.lower().split("_")
</span><span class="cx"> def cap(word):
</span><del>- if word.lower() == 'id':
</del><ins>+ if word.lower() in ('id', 'guid'):
</ins><span class="cx"> return word.upper()
</span><span class="cx"> else:
</span><span class="cx"> return word.capitalize()
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwextenterprisedalsyntaxpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/syntax.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/syntax.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/twext/enterprise/dal/syntax.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -1816,15 +1816,21 @@
</span><span class="cx"> params = []
</span><span class="cx"> for parameter in self.parameters:
</span><span class="cx"> if isinstance(parameter, Parameter):
</span><ins>+ if parameter.name not in kw:
+ # TODO UNIT TEST
+ raise DALError("Missing parameter: {p!r} fragment={f!r}, kw={kw!r}".format(p=parameter.name,
+ f=self.text, kw=kw))
+ parameterValue = kw[parameter.name]
+
</ins><span class="cx"> if parameter.count is not None:
</span><del>- if parameter.count != len(kw[parameter.name]):
</del><ins>+ if parameter.count != len(parameterValue):
</ins><span class="cx"> raise DALError("Number of place holders does not match number of items to bind")
</span><del>- for item in kw[parameter.name]:
</del><ins>+ for item in parameterValue:
</ins><span class="cx"> params.append(item)
</span><span class="cx"> else:
</span><del>- params.append(kw[parameter.name])
</del><ins>+ params.append(parameterValue)
</ins><span class="cx"> else:
</span><del>- params.append(parameter)
</del><ins>+ params.append(parameterValue)
</ins><span class="cx"> return SQLFragment(self.text, params)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwextwhodelegatespy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/sagen/groupcacher/twext/who/delegates.py (0 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twext/who/delegates.py (rev 0)
+++ CalendarServer/branches/users/sagen/groupcacher/twext/who/delegates.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -0,0 +1,86 @@
</span><ins>+# -*- test-case-name: twext.who.test.test_delegates -*-
+##
+# Copyright (c) 2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Delegate assignments
+"""
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twext.who.idirectory import RecordType
+
+from twext.python.log import Logger
+log = Logger()
+
+
+@inlineCallbacks
+def addDelegate(txn, delegator, delegate, readWrite):
+ """
+ Args are records
+ """
+ if delegate.recordType == RecordType.group:
+ # find the groupID
+ results = (yield txn.groupByGUID(delegate.guid))
+ while not results:
+ # need to add the group to the groups table so we have a groupID
+
+ # TODO: is there a better pattern for this?
+ yield txn.addGroup(delegate.guid, delegate.fullNames[0], "")
+ results = (yield txn.groupByGUID(delegate.guid))
+
+ groupID = results[0][0]
+ yield txn.addDelegate(delegator.guid, groupID,
+ 1 if readWrite else 0, True)
+ else:
+ yield txn.addDelegate(delegator.guid, delegate.guid,
+ 1 if readWrite else 0, False)
+
+def removeDelegate(txn, delegator, delegate, readWrite):
+ """
+ Args are records
+ """
+ return txn.removeDelegate(delegator.guid, delegate.guid,
+ 1 if readWrite else 0, delegate.recordType==RecordType.group)
+
+@inlineCallbacks
+def delegatesOf(txn, delegator, readWrite):
+ """
+ Args are records
+ """
+ records = []
+ directory = delegator.service
+ results = (yield txn.delegates(delegator.guid, 1 if readWrite else 0))
+ for row in results:
+ if row[0] != delegator.guid:
+ record = (yield directory.recordWithGUID(row[0]))
+ if record is not None:
+ records.append(record)
+ returnValue(records)
+
+@inlineCallbacks
+def delegateFor(txn, delegate, readWrite):
+ """
+ Args are records
+ """
+ records = []
+ directory = delegate.service
+ results = (yield txn.delegators(delegate.guid, 1 if readWrite else 0))
+ for row in results:
+ if row[0] != delegate.guid:
+ record = (yield directory.recordWithGUID(row[0]))
+ if record is not None:
+ records.append(record)
+ returnValue(records)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwextwhogroupspy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/sagen/groupcacher/twext/who/groups.py (0 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twext/who/groups.py (rev 0)
+++ CalendarServer/branches/users/sagen/groupcacher/twext/who/groups.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -0,0 +1,284 @@
</span><ins>+# -*- test-case-name: twext.who.test.test_groups -*-
+##
+# Copyright (c) 2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Group membership caching
+"""
+
+import datetime
+import hashlib
+from twext.enterprise.dal.record import fromTable
+from twext.enterprise.queue import WorkItem, PeerConnectionPool
+from txdav.common.datastore.sql_tables import schema
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twext.enterprise.dal.syntax import Delete
+from twext.who.idirectory import RecordType
+
+from twext.python.log import Logger
+log = Logger()
+
+
+class GroupCacherPollingWork(WorkItem,
+ fromTable(schema.GROUP_CACHER_POLLING_WORK)):
+
+ group = "group_cacher_polling"
+
+ @inlineCallbacks
+ def doWork(self):
+
+ # Delete all other work items
+ yield Delete(From=self.table, Where=None).on(self.transaction)
+
+ groupCacher = getattr(self.transaction, "_groupCacher", None)
+ if groupCacher is not None:
+
+ # Schedule next update
+ notBefore = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=groupCacher.updateSeconds))
+ log.debug("Scheduling next group cacher update: %s" % (notBefore,))
+ yield self.transaction.enqueue(GroupCacherPollingWork,
+ notBefore=notBefore)
+
+ try:
+ groupCacher.update(self.transaction)
+ except Exception, e:
+ log.error("Failed to update group membership cache (%s)" % (e,))
+
+ else:
+ notBefore = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=10))
+ log.debug("Rescheduling group cacher update: %s" % (notBefore,))
+ yield self.transaction.enqueue(GroupCacherPollingWork,
+ notBefore=notBefore)
+
+
+@inlineCallbacks
+def scheduleNextGroupCachingUpdate(store, seconds):
+ txn = store.newTransaction()
+ notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
+ log.debug("Scheduling next group cacher update: %s" % (notBefore,))
+ wp = (yield txn.enqueue(GroupCacherPollingWork, notBefore=notBefore))
+ yield txn.commit()
+ returnValue(wp)
+
+
+def schedulePolledGroupCachingUpdate(store):
+ """
+ Schedules a group caching update work item in "the past" so PeerConnectionPool's
+ overdue-item logic picks it up quickly.
+ """
+ seconds = -PeerConnectionPool.queueProcessTimeout
+ return scheduleNextGroupCachingUpdate(store, seconds)
+
+
+class GroupRefreshWork(WorkItem, fromTable(schema.GROUP_REFRESH_WORK)):
+
+ group = property(lambda self: self.groupGUID)
+
+ @inlineCallbacks
+ def doWork(self):
+
+ # Delete all other work items
+ yield Delete(From=self.table, Where=None).on(self.transaction)
+
+ groupCacher = getattr(self.transaction, "_groupCacher", None)
+ if groupCacher is not None:
+
+ try:
+ groupCacher.refreshGroup(self.transaction, self.groupGUID)
+ except Exception, e:
+ log.error("Failed to refresh group {group} {err}",
+ group=self.groupGUID, err=e)
+
+ else:
+ notBefore = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=10))
+ log.debug("Rescheduling group refresh for {group}: {when}",
+ group=self.groupGUID, when=notBefore)
+ yield self.transaction.enqueue(GroupRefreshWork,
+ groupGUID=self.groupGUID, notBefore=notBefore)
+
+
+class GroupAttendeeReconciliationWork(WorkItem, fromTable(schema.GROUP_ATTENDEE_RECONCILIATION_WORK)):
+ pass
+
+
+@inlineCallbacks
+def _expandedMembers(record, members=None, records=None):
+ """
+ Return the expanded set of member records. Intermediate groups are not returned
+ in the results, but their members are.
+ """
+ if members is None:
+ members = set()
+ if records is None:
+ records = set()
+
+ if record not in records:
+ records.add(record)
+ for member in (yield record.members()):
+ if member not in records:
+ if member.recordType != RecordType.group:
+ members.add(member)
+ yield _expandedMembers(member, members, records)
+
+ returnValue(members)
+
+
+class GroupCacher(object):
+ log = Logger()
+
+ def __init__(self, proxyDB, directory, store, updateSeconds,
+ useExternalProxies=False, externalProxiesSource=None):
+ self.proxyDB = proxyDB
+ self.directory = directory
+ self.store = store
+ self.updateSeconds = updateSeconds
+ self.useExternalProxies = useExternalProxies
+ if useExternalProxies and externalProxiesSource is None:
+ externalProxiesSource = self.directory.getExternalProxyAssignments
+ self.externalProxiesSource = externalProxiesSource
+
+
+ @inlineCallbacks
+ def update(self, txn):
+ # Pull in external proxy assignments and stick in proxy db
+ # Figure out which groups matter
+ groupGUIDs = yield self.groupsToRefresh()
+ # For each of those groups, create a per-group refresh work item
+ for groupGUID in groupGUIDs:
+ notBefore = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=1))
+ yield txn.enqueue(GroupRefreshWork,
+ groupGUID=groupGUID, notBefore=notBefore)
+
+ pass
+
+
+ @inlineCallbacks
+ def refreshGroup(self, txn, groupGUID):
+ # Does the work of a per-group refresh work item
+ # Faults in the flattened membership of a group, as GUIDs
+ # and updates the GROUP_MEMBERSHIP table
+ record = (yield self.directory.recordWithGUID(groupGUID))
+ membershipHashContent = hashlib.md5()
+ members = (yield _expandedMembers(record))
+ members = list(members)
+ members.sort(cmp=lambda x,y: cmp(x.guid, y.guid))
+ for member in members:
+ membershipHashContent.update(member.guid)
+ membershipHash = membershipHashContent.hexdigest()
+ results = (yield txn.groupByGUID(groupGUID))
+ if not results:
+ # Group is not yet in the DB
+ cachedName = ""
+ cachedMembershipHash = ""
+ addGroup = True
+ else:
+ groupID, cachedName, cachedMembershipHash = results[0]
+ addGroup = False
+
+ if cachedMembershipHash != membershipHash:
+ membershipChanged = True
+ self.log.debug("Group '{group}' changed", group=record.fullNames[0])
+ else:
+ membershipChanged = False
+
+ if addGroup:
+ yield txn.addGroup(groupGUID, record.fullNames[0], membershipHash)
+ else:
+ yield txn.updateGroup(groupGUID, record.fullNames[0],
+ membershipHash)
+
+ results = (yield txn.groupByGUID(groupGUID))
+ if len(results) == 1:
+ groupID, name, cachedMembershipHash = results[0]
+ else:
+ self.log.error("Multiple group entries for {guid}", guid=groupGUID)
+
+ if membershipChanged:
+ newMemberGUIDs = set()
+ for member in members:
+ newMemberGUIDs.add(member.guid)
+ yield self.synchronizeMembers(txn, groupID, newMemberGUIDs)
+
+ yield self.scheduleEventReconciliations(txn, groupID)
+
+
+ @inlineCallbacks
+ def synchronizeMembers(self, txn, groupID, newMemberGUIDs):
+ numRemoved = numAdded = 0
+ cachedMemberGUIDs = set()
+ results = (yield txn.membersOfGroup(groupID))
+ for row in results:
+ cachedMemberGUIDs.add(row[0])
+
+ for memberGUID in cachedMemberGUIDs:
+ if memberGUID not in newMemberGUIDs:
+ numRemoved += 1
+ yield txn.removeMemberFromGroup(memberGUID, groupID)
+
+ for memberGUID in newMemberGUIDs:
+ if memberGUID not in cachedMemberGUIDs:
+ numAdded += 1
+ yield txn.addMemberToGroup(memberGUID, groupID)
+
+ returnValue((numAdded, numRemoved))
+
+
+ @inlineCallbacks
+ def cachedMembers(self, txn, groupID):
+ members = set()
+ results = (yield txn.membersOfGroup(groupID))
+ for row in results:
+ record = (yield self.directory.recordWithGUID(row[0]))
+ if record is not None:
+ members.add(record)
+ returnValue(members)
+
+
+
+
+ # @inlineCallbacks
+ def scheduleEventReconciliations(self, txn, groupID):
+ """
+ Find all events who have this groupID as an attendee and create
+ work items for them.
+ """
+ return succeed(None)
+
+
+
+ @inlineCallbacks
+ def groupsToRefresh(self):
+ delegatedGUIDs = set((yield self.proxyDB.getAllMembers()))
+ self.log.info("There are %d proxies" % (len(delegatedGUIDs),))
+ self.log.info("Retrieving group hierarchy from directory")
+
+ # "groups" maps a group to its members; the keys and values consist
+ # of whatever directory attribute is used to refer to members. The
+ # attribute value comes from record.cachedGroupsAlias().
+ # "aliases" maps the record.cachedGroupsAlias() value for a group
+ # back to the group's guid.
+ groups, aliases = (yield self.getGroups(guids=delegatedGUIDs))
+ groupGUIDs = set(aliases.keys())
+ self.log.info("%d groups retrieved from the directory" %
+ (len(groupGUIDs),))
+
+ delegatedGUIDs = delegatedGUIDs.intersection(groupGUIDs)
+ self.log.info("%d groups are proxies" % (len(delegatedGUIDs),))
+ returnValue(delegatedGUIDs)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwextwhotesttest_delegatespy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_delegates.py (0 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_delegates.py (rev 0)
+++ CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_delegates.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -0,0 +1,186 @@
</span><ins>+##
+# Copyright (c) 2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Delegates implementation tests
+"""
+
+from twext.who.delegates import (
+ addDelegate, removeDelegate, delegatesOf, delegateFor
+)
+from twext.who.groups import GroupCacher
+from twext.who.test.test_xml import xmlService
+from twisted.internet.defer import inlineCallbacks
+from twistedcaldav.test.util import StoreTestCase
+
+class DelegationTest(StoreTestCase):
+
+ @inlineCallbacks
+ def setUp(self):
+ yield super(DelegationTest, self).setUp()
+ self.xmlService = xmlService(self.mktemp(), xmlData=testXMLConfig)
+ self.groupCacher = GroupCacher(
+ None,
+ self.xmlService,
+ None,
+ 0
+ )
+
+
+ @inlineCallbacks
+ def test_directDelegation(self):
+ store = self.storeUnderTest()
+ txn = store.newTransaction()
+
+ delegator = yield self.xmlService.recordWithUID("__wsanchez__")
+ delegate1 = yield self.xmlService.recordWithUID("__sagen__")
+ delegate2 = yield self.xmlService.recordWithUID("__cdaboo__")
+
+ # Add 1 delegate
+ yield addDelegate(txn, delegator, delegate1, True)
+ delegates = (yield delegatesOf(txn, delegator, True))
+ self.assertEquals(["sagen"], [d.shortNames[0] for d in delegates])
+ delegators = (yield delegateFor(txn, delegate1, True))
+ self.assertEquals(["wsanchez"], [d.shortNames[0] for d in delegators])
+
+ # Add another delegate
+ yield addDelegate(txn, delegator, delegate2, True)
+ delegates = (yield delegatesOf(txn, delegator, True))
+ self.assertEquals(set(["sagen", "cdaboo"]),
+ set([d.shortNames[0] for d in delegates]))
+ delegators = (yield delegateFor(txn, delegate2, True))
+ self.assertEquals(["wsanchez"], [d.shortNames[0] for d in delegators])
+
+ # Remove 1 delegate
+ yield removeDelegate(txn, delegator, delegate1, True)
+ delegates = (yield delegatesOf(txn, delegator, True))
+ self.assertEquals(["cdaboo"], [d.shortNames[0] for d in delegates])
+ delegators = (yield delegateFor(txn, delegate1, True))
+ self.assertEquals(0, len(delegators))
+
+ # Remove the other delegate
+ yield removeDelegate(txn, delegator, delegate2, True)
+ delegates = (yield delegatesOf(txn, delegator, True))
+ self.assertEquals(0, len(delegates))
+ delegators = (yield delegateFor(txn, delegate2, True))
+ self.assertEquals(0, len(delegators))
+
+
+ @inlineCallbacks
+ def test_indirectDelegation(self):
+ store = self.storeUnderTest()
+ txn = store.newTransaction()
+
+ delegator = yield self.xmlService.recordWithUID("__wsanchez__")
+ group1 = yield self.xmlService.recordWithUID("__top_group_1__")
+ delegate1 = yield self.xmlService.recordWithUID("__sagen__")
+
+ # Add group delegate, but before the group membership has been
+ # pulled in
+ yield addDelegate(txn, delegator, group1, True)
+ delegates = (yield delegatesOf(txn, delegator, True))
+ self.assertEquals(0, len(delegates))
+
+ # Now refresh the group and there will be 3 delegates (contained
+ # within 2 nested groups)
+ guid = "49b350c69611477b94d95516b13856ab"
+ yield self.groupCacher.refreshGroup(txn, guid)
+ delegates = (yield delegatesOf(txn, delegator, True))
+ self.assertEquals(set(["sagen", "cdaboo", "glyph"]),
+ set([d.shortNames[0] for d in delegates]))
+ delegators = (yield delegateFor(txn, delegate1, True))
+ self.assertEquals(["wsanchez"], [d.shortNames[0] for d in delegators])
+
+
+
+
+testXMLConfig = """<?xml version="1.0" encoding="utf-8"?>
+
+<directory realm="xyzzy">
+
+ <record type="user">
+ <uid>__wsanchez__</uid>
+ <guid>3BDCB954-84D5-4F6D-8035-EAC19A6D6E1F</guid>
+ <short-name>wsanchez</short-name>
+ <short-name>wilfredo_sanchez</short-name>
+ <full-name>Wilfredo Sanchez</full-name>
+ <password>zehcnasw</password>
+ <email>wsanchez@bitbucket.calendarserver.org</email>
+ <email>wsanchez@devnull.twistedmatrix.com</email>
+ </record>
+
+ <record type="user">
+ <uid>__glyph__</uid>
+ <guid>9064DF91-1DBC-4E07-9C2B-6839B0953876</guid>
+ <short-name>glyph</short-name>
+ <full-name>Glyph Lefkowitz</full-name>
+ <password>hpylg</password>
+ <email>glyph@bitbucket.calendarserver.org</email>
+ <email>glyph@devnull.twistedmatrix.com</email>
+ </record>
+
+ <record type="user">
+ <uid>__sagen__</uid>
+ <guid>4AD155CB-AE9B-475F-986C-E08A7537893E</guid>
+ <short-name>sagen</short-name>
+ <full-name>Morgen Sagen</full-name>
+ <password>negas</password>
+ <email>sagen@bitbucket.calendarserver.org</email>
+ <email>shared@example.com</email>
+ </record>
+
+ <record type="user">
+ <uid>__cdaboo__</uid>
+ <guid>7D45CB10-479E-456B-B54D-528958C5734B</guid>
+ <short-name>cdaboo</short-name>
+ <full-name>Cyrus Daboo</full-name>
+ <password>suryc</password>
+ <email>cdaboo@bitbucket.calendarserver.org</email>
+ </record>
+
+ <record type="user">
+ <uid>__dre__</uid>
+ <guid>CFC88493-DBFF-42B9-ADC7-9B3DA0B0769B</guid>
+ <short-name>dre</short-name>
+ <full-name>Andre LaBranche</full-name>
+ <password>erd</password>
+ <email>dre@bitbucket.calendarserver.org</email>
+ <email>shared@example.com</email>
+ </record>
+
+ <record type="group">
+ <uid>__top_group_1__</uid>
+ <guid>49B350C6-9611-477B-94D9-5516B13856AB</guid>
+ <short-name>top-group-1</short-name>
+ <full-name>Top Group 1</full-name>
+ <email>topgroup1@example.com</email>
+ <member-uid>__wsanchez__</member-uid>
+ <member-uid>__glyph__</member-uid>
+ <member-uid>__sub_group_1__</member-uid>
+ </record>
+
+ <record type="group">
+ <uid>__sub_group_1__</uid>
+ <guid>86144F73-345A-4097-82F1-B782672087C7</guid>
+ <short-name>sub-group-1</short-name>
+ <full-name>Sub Group 1</full-name>
+ <email>subgroup1@example.com</email>
+ <member-uid>__sagen__</member-uid>
+ <member-uid>__cdaboo__</member-uid>
+ </record>
+
+</directory>
+"""
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwextwhotesttest_groupspy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_groups.py (0 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_groups.py (rev 0)
+++ CalendarServer/branches/users/sagen/groupcacher/twext/who/test/test_groups.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -0,0 +1,205 @@
</span><ins>+##
+# Copyright (c) 2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Group membership caching implementation tests
+"""
+
+from twext.who.groups import GroupCacher, _expandedMembers
+from twext.who.test.test_xml import xmlService
+from twext.who.idirectory import RecordType
+from twisted.internet.defer import inlineCallbacks
+from twistedcaldav.test.util import StoreTestCase
+
+class GroupCacherTest(StoreTestCase):
+
+ @inlineCallbacks
+ def setUp(self):
+ yield super(GroupCacherTest, self).setUp()
+ self.xmlService = xmlService(self.mktemp(), xmlData=testXMLConfig)
+ self.groupCacher = GroupCacher(
+ None,
+ self.xmlService,
+ None,
+ 0
+ )
+
+
+ @inlineCallbacks
+ def test_expandedMembers(self):
+ record = yield self.xmlService.recordWithUID("__top_group_1__")
+ memberUIDs = set()
+ for member in (yield _expandedMembers(record)):
+ memberUIDs.add(member.uid)
+ self.assertEquals(memberUIDs, set(["__cdaboo__",
+ "__glyph__", "__sagen__", "__wsanchez__"]))
+
+ record = yield self.xmlService.recordWithUID("__sagen__")
+ members = yield _expandedMembers(record)
+ self.assertEquals(0, len(list(members)))
+
+
+ @inlineCallbacks
+ def test_refreshGroup(self):
+ """
+ Verify refreshGroup() adds a group to the Groups table with the
+ expected membership hash value and members
+ """
+
+ store = self.storeUnderTest()
+ txn = store.newTransaction()
+
+ guid = "49b350c69611477b94d95516b13856ab"
+ yield self.groupCacher.refreshGroup(txn, guid)
+
+ groupID, name, membershipHash = (yield txn.groupByGUID(guid))[0]
+ self.assertEquals(membershipHash, "e90052eb63d47f32d5b03df0073f7854")
+
+ groupGUID, name, membershipHash = (yield txn.groupByID(groupID))[0]
+ self.assertEquals(groupGUID, guid)
+ self.assertEquals(name, "Top Group 1")
+ self.assertEquals(membershipHash, "e90052eb63d47f32d5b03df0073f7854")
+
+ results = (yield txn.membersOfGroup(groupID))
+ for row in results:
+ print row[0]
+
+ records = (yield self.groupCacher.cachedMembers(txn, groupID))
+ self.assertEquals(
+ set([r.shortNames[0] for r in records]),
+ set(["wsanchez", "cdaboo", "glyph", "sagen"])
+ )
+ print records
+
+
+ @inlineCallbacks
+ def test_synchronizeMembers(self):
+ """
+ After loading in a group via refreshGroup(), pass new member sets to
+ synchronizeMembers() and verify members are added and removed as
+ expected
+ """
+
+ store = self.storeUnderTest()
+ txn = store.newTransaction()
+
+ # Refresh the group so it's assigned a group_id
+ guid = "49b350c69611477b94d95516b13856ab"
+ yield self.groupCacher.refreshGroup(txn, guid)
+ groupID, name, membershipHash = (yield txn.groupByGUID(guid))[0]
+
+ # Remove two members, and add one member
+ newSet = set()
+ for name in ("wsanchez", "cdaboo", "dre"):
+ record = (yield self.xmlService.recordWithShortName(RecordType.user,
+ name))
+ newSet.add(record.guid)
+ numAdded, numRemoved = (yield self.groupCacher.synchronizeMembers(txn,
+ groupID, newSet))
+ self.assertEquals(numAdded, 1)
+ self.assertEquals(numRemoved, 2)
+ records = (yield self.groupCacher.cachedMembers(txn, groupID))
+ self.assertEquals(
+ set([r.shortNames[0] for r in records]),
+ set(["wsanchez", "cdaboo", "dre"])
+ )
+
+ # Remove all members
+ numAdded, numRemoved = (yield self.groupCacher.synchronizeMembers(txn,
+ groupID, set()))
+ self.assertEquals(numAdded, 0)
+ self.assertEquals(numRemoved, 3)
+ records = (yield self.groupCacher.cachedMembers(txn, groupID))
+ self.assertEquals(len(records), 0)
+
+
+testXMLConfig = """<?xml version="1.0" encoding="utf-8"?>
+
+<directory realm="xyzzy">
+
+ <record type="user">
+ <uid>__wsanchez__</uid>
+ <guid>3BDCB954-84D5-4F6D-8035-EAC19A6D6E1F</guid>
+ <short-name>wsanchez</short-name>
+ <short-name>wilfredo_sanchez</short-name>
+ <full-name>Wilfredo Sanchez</full-name>
+ <password>zehcnasw</password>
+ <email>wsanchez@bitbucket.calendarserver.org</email>
+ <email>wsanchez@devnull.twistedmatrix.com</email>
+ </record>
+
+ <record type="user">
+ <uid>__glyph__</uid>
+ <guid>9064DF91-1DBC-4E07-9C2B-6839B0953876</guid>
+ <short-name>glyph</short-name>
+ <full-name>Glyph Lefkowitz</full-name>
+ <password>hpylg</password>
+ <email>glyph@bitbucket.calendarserver.org</email>
+ <email>glyph@devnull.twistedmatrix.com</email>
+ </record>
+
+ <record type="user">
+ <uid>__sagen__</uid>
+ <guid>4AD155CB-AE9B-475F-986C-E08A7537893E</guid>
+ <short-name>sagen</short-name>
+ <full-name>Morgen Sagen</full-name>
+ <password>negas</password>
+ <email>sagen@bitbucket.calendarserver.org</email>
+ <email>shared@example.com</email>
+ </record>
+
+ <record type="user">
+ <uid>__cdaboo__</uid>
+ <guid>7D45CB10-479E-456B-B54D-528958C5734B</guid>
+ <short-name>cdaboo</short-name>
+ <full-name>Cyrus Daboo</full-name>
+ <password>suryc</password>
+ <email>cdaboo@bitbucket.calendarserver.org</email>
+ </record>
+
+ <record type="user">
+ <uid>__dre__</uid>
+ <guid>CFC88493-DBFF-42B9-ADC7-9B3DA0B0769B</guid>
+ <short-name>dre</short-name>
+ <full-name>Andre LaBranche</full-name>
+ <password>erd</password>
+ <email>dre@bitbucket.calendarserver.org</email>
+ <email>shared@example.com</email>
+ </record>
+
+ <record type="group">
+ <uid>__top_group_1__</uid>
+ <guid>49B350C6-9611-477B-94D9-5516B13856AB</guid>
+ <short-name>top-group-1</short-name>
+ <full-name>Top Group 1</full-name>
+ <email>topgroup1@example.com</email>
+ <member-uid>__wsanchez__</member-uid>
+ <member-uid>__glyph__</member-uid>
+ <member-uid>__sub_group_1__</member-uid>
+ </record>
+
+ <record type="group">
+ <uid>__sub_group_1__</uid>
+ <guid>86144F73-345A-4097-82F1-B782672087C7</guid>
+ <short-name>sub-group-1</short-name>
+ <full-name>Sub Group 1</full-name>
+ <email>subgroup1@example.com</email>
+ <member-uid>__sagen__</member-uid>
+ <member-uid>__cdaboo__</member-uid>
+ </record>
+
+</directory>
+"""
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertwistedcaldavdirectorydirectorypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/twistedcaldav/directory/directory.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/twistedcaldav/directory/directory.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/twistedcaldav/directory/directory.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -31,9 +31,6 @@
</span><span class="cx">
</span><span class="cx"> from plistlib import readPlistFromString
</span><span class="cx">
</span><del>-from twext.enterprise.dal.record import fromTable
-from twext.enterprise.dal.syntax import Delete
-from twext.enterprise.queue import WorkItem, PeerConnectionPool
</del><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twext.web2.dav.auth import IPrincipalCredentials
</span><span class="cx"> from twext.web2.dav.util import joinURL
</span><span class="lines">@@ -52,7 +49,6 @@
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.icalendardirectoryservice import ICalendarStoreDirectoryService, \
</span><span class="cx"> ICalendarStoreDirectoryRecord
</span><del>-from txdav.common.datastore.sql_tables import schema
</del><span class="cx">
</span><span class="cx"> from xml.parsers.expat import ExpatError
</span><span class="cx">
</span><span class="lines">@@ -1038,62 +1034,6 @@
</span><span class="cx"> returnValue((fast, len(members), len(changedMembers)))
</span><span class="cx">
</span><span class="cx">
</span><del>-
-class GroupCacherPollingWork(WorkItem, fromTable(schema.GROUP_CACHER_POLLING_WORK)):
-
- group = "group_cacher_polling"
-
- @inlineCallbacks
- def doWork(self):
-
- # Delete all other work items
- yield Delete(From=self.table, Where=None).on(self.transaction)
-
- groupCacher = getattr(self.transaction, "_groupCacher", None)
- if groupCacher is not None:
-
- # Schedule next update
- notBefore = (datetime.datetime.utcnow() +
- datetime.timedelta(seconds=groupCacher.updateSeconds))
- log.debug("Scheduling next group cacher update: %s" % (notBefore,))
- yield self.transaction.enqueue(GroupCacherPollingWork,
- notBefore=notBefore)
-
- try:
- groupCacher.updateCache()
- except Exception, e:
- log.error("Failed to update group membership cache (%s)" % (e,))
-
- else:
- notBefore = (datetime.datetime.utcnow() +
- datetime.timedelta(seconds=10))
- log.debug("Rescheduling group cacher update: %s" % (notBefore,))
- yield self.transaction.enqueue(GroupCacherPollingWork,
- notBefore=notBefore)
-
-
-
-@inlineCallbacks
-def scheduleNextGroupCachingUpdate(store, seconds):
- txn = store.newTransaction()
- notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
- log.debug("Scheduling next group cacher update: %s" % (notBefore,))
- wp = (yield txn.enqueue(GroupCacherPollingWork, notBefore=notBefore))
- yield txn.commit()
- returnValue(wp)
-
-
-
-def schedulePolledGroupCachingUpdate(store):
- """
- Schedules a group caching update work item in "the past" so PeerConnectionPool's
- overdue-item logic picks it up quickly.
- """
- seconds = -PeerConnectionPool.queueProcessTimeout
- return scheduleNextGroupCachingUpdate(store, seconds)
-
-
-
</del><span class="cx"> def diffAssignments(old, new):
</span><span class="cx"> """
</span><span class="cx"> Compare two proxy assignment lists and return their differences in the form of
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql.py (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql.py 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql.py 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -80,6 +80,7 @@
</span><span class="cx">
</span><span class="cx"> import sys
</span><span class="cx"> import time
</span><ins>+import datetime
</ins><span class="cx">
</span><span class="cx"> current_sql_schema = getModule(__name__).filePath.sibling("sql_schema").child("current.sql").getContent()
</span><span class="cx">
</span><span class="lines">@@ -868,7 +869,282 @@
</span><span class="cx">
</span><span class="cx"> # End of IMIP
</span><span class="cx">
</span><ins>+ # Groups
</ins><span class="cx">
</span><ins>+ @classproperty
+ def _addGroupQuery(cls): #@NoSelf
+ gr = schema.GROUPS
+ return Insert({gr.NAME: Parameter("name"),
+ gr.GROUP_GUID: Parameter("groupGUID"),
+ gr.MEMBERSHIP_HASH: Parameter("membershipHash")})
+
+
+ @classproperty
+ def _updateGroupQuery(cls): #@NoSelf
+ gr = schema.GROUPS
+ return Update({gr.MEMBERSHIP_HASH: Parameter("membershipHash"),
+ gr.NAME: Parameter("name"), gr.MODIFIED: Parameter("timestamp")},
+ Where=(gr.GROUP_GUID == Parameter("groupGUID")))
+
+
+ @classproperty
+ def _groupByGUID(cls): #@NoSelf
+ gr = schema.GROUPS
+ return Select([gr.GROUP_ID, gr.NAME, gr.MEMBERSHIP_HASH], From=gr,
+ Where=(
+ gr.GROUP_GUID == Parameter("groupGUID")
+ )
+ )
+
+
+ @classproperty
+ def _groupByID(cls): #@NoSelf
+ gr = schema.GROUPS
+ return Select([gr.GROUP_GUID, gr.NAME, gr.MEMBERSHIP_HASH], From=gr,
+ Where=(
+ gr.GROUP_ID == Parameter("groupID")
+ )
+ )
+
+
+ @classproperty
+ def _deleteGroup(cls): #@NoSelf
+ gr = schema.GROUPS
+ return Delete(From=gr,
+ Where=(gr.GROUP_ID == Parameter("groupID")))
+
+
+ def addGroup(self, groupGUID, name, membershipHash):
+ return self._addGroupQuery.on(self, name=name,
+ groupGUID=groupGUID, membershipHash=membershipHash)
+
+
+ def updateGroup(self, groupGUID, name, membershipHash):
+ timestamp = datetime.datetime.utcnow()
+ return self._updateGroupQuery.on(self, name=name,
+ groupGUID=groupGUID, timestamp=timestamp,
+ membershipHash=membershipHash)
+
+
+ def groupByGUID(self, groupGUID):
+ return self._groupByGUID.on(self, groupGUID=groupGUID)
+
+
+ def groupByID(self, groupID):
+ return self._groupByID.on(self, groupID=groupID)
+
+
+ def deleteGroup(self, groupID):
+ return self._deleteGroup.on(self, groupID=groupID)
+
+ # End of Groups
+
+
+ # Group Members
+
+ @classproperty
+ def _addMemberToGroupQuery(cls): #@NoSelf
+ gm = schema.GROUP_MEMBERSHIP
+ return Insert({gm.GROUP_ID: Parameter("groupID"),
+ gm.MEMBER_GUID: Parameter("memberGUID")})
+ @classproperty
+ def _removeMemberFromGroupQuery(cls): #@NoSelf
+ gm = schema.GROUP_MEMBERSHIP
+ return Delete(From=gm,
+ Where=(gm.GROUP_ID == Parameter("groupID")).And(
+ gm.MEMBER_GUID == Parameter("memberGUID")))
+
+ @classproperty
+ def _selectGroupMembersQuery(cls): #@NoSelf
+ gm = schema.GROUP_MEMBERSHIP
+ return Select([gm.MEMBER_GUID], From=gm,
+ Where=(
+ gm.GROUP_ID == Parameter("groupID")
+ )
+ )
+
+ def addMemberToGroup(self, memberGUID, groupID):
+ return self._addMemberToGroupQuery.on(self,
+ groupID=groupID, memberGUID=memberGUID)
+
+ def removeMemberFromGroup(self, memberGUID, groupID):
+ return self._removeMemberFromGroupQuery.on(self,
+ groupID=groupID, memberGUID=memberGUID)
+
+ def membersOfGroup(self, groupID):
+ return self._selectGroupMembersQuery.on(self, groupID=groupID)
+
+ # End of Group Members
+
+ # Delegates
+
+ @classproperty
+ def _addDelegateQuery(cls): #@NoSelf
+ de = schema.DELEGATES
+ return Insert({de.DELEGATOR: Parameter("delegator"),
+ de.DELEGATE: Parameter("delegate"),
+ de.READ_WRITE: Parameter("readWrite"),
+ })
+
+ @classproperty
+ def _addDelegateGroupQuery(cls): #@NoSelf
+ ds = schema.DELEGATE_GROUPS
+ return Insert({ds.DELEGATOR: Parameter("delegator"),
+ ds.GROUP_ID: Parameter("groupID"),
+ ds.READ_WRITE: Parameter("readWrite"),
+ })
+
+ @classproperty
+ def _removeDelegateQuery(cls): #@NoSelf
+ de = schema.DELEGATES
+ return Delete(From=de,
+ Where=(de.DELEGATOR == Parameter("delegator")).And(
+ de.DELEGATE == Parameter("delegate")).And(
+ de.READ_WRITE == Parameter("readWrite"))
+ )
+
+ @classproperty
+ def _removeDelegateGroupQuery(cls): #@NoSelf
+ ds = schema.DELEGATE_GROUPS
+ return Delete(From=ds,
+ Where=(ds.DELEGATOR == Parameter("delegator")).And(
+ ds.GROUP_ID == Parameter("groupID")).And(
+ ds.READ_WRITE == Parameter("readWrite"))
+ )
+
+ @classproperty
+ def _selectDelegatesQuery(cls): #@NoSelf
+ de = schema.DELEGATES
+ return Select([de.DELEGATE], From=de,
+ Where=(de.DELEGATOR == Parameter("delegator")).And(
+ de.READ_WRITE == Parameter("readWrite"))
+ )
+
+ @classproperty
+ def _selectDelegateGroupsQuery(cls): #@NoSelf
+ ds = schema.DELEGATE_GROUPS
+ return Select([ds.GROUP_ID], From=ds,
+ Where=(
+ ds.DELEGATOR == Parameter("delegator")).And(
+ ds.READ_WRITE == Parameter("readWrite"))
+ )
+
+ @classproperty
+ def _selectDirectDelegatorsQuery(cls): #@NoSelf
+ de = schema.DELEGATES
+ return Select([de.DELEGATOR], From=de,
+ Where=(
+ de.DELEGATE == Parameter("delegate")).And(
+ de.READ_WRITE == Parameter("readWrite"))
+ )
+
+ """
+ @classproperty
+ def _selectDelegatorsGroupsQuery(cls): #@NoSelf
+ ds = schema.DELEGATE_GROUPS
+ return Select([ds.DELEGATOR], From=ds,
+ Where=(
+ ds.GROUP_ID == Parameter("groupID").And(
+ ds.READ_WRITE == Parameter("readWrite"))
+ )
+ )
+ """
+
+ @classproperty
+ def _selectIndirectDelegatorsQuery(cls): #@NoSelf
+ dg = schema.DELEGATE_GROUPS
+ gm = schema.GROUP_MEMBERSHIP
+
+ return Select(
+ [dg.DELEGATOR],
+ From=dg,
+ Where=(
+ dg.GROUP_ID.In(
+ Select(
+ [gm.GROUP_ID],
+ From=gm,
+ Where=(gm.MEMBER_GUID == Parameter("delegate"))
+ )
+ ).And(
+ dg.READ_WRITE == Parameter("readWrite")
+ )
+ )
+ )
+
+ @classproperty
+ def _selectIndirectDelegatesQuery(cls): #@NoSelf
+ dg = schema.DELEGATE_GROUPS
+ gm = schema.GROUP_MEMBERSHIP
+
+ return Select(
+ [gm.MEMBER_GUID],
+ From=gm,
+ Where=(
+ gm.GROUP_ID.In(
+ Select(
+ [dg.GROUP_ID],
+ From=dg,
+ Where=(dg.DELEGATOR == Parameter("delegator")).And(
+ dg.READ_WRITE == Parameter("readWrite"))
+ )
+ )
+ )
+ )
+
+ def addDelegate(self, delegator, delegate, readWrite, isGroup):
+ if isGroup:
+ return self._addDelegateGroupQuery.on(self, delegator=delegator,
+ groupID=delegate, readWrite=readWrite)
+ else:
+ return self._addDelegateQuery.on(self, delegator=delegator,
+ delegate=delegate, readWrite=readWrite)
+
+ def removeDelegate(self, delegator, delegate, readWrite, isGroup):
+ if isGroup:
+ return self._removeDelegateGroupQuery.on(self, delegator=delegator,
+ groupID=delegate, readWrite=readWrite)
+ else:
+ return self._removeDelegateQuery.on(self, delegator=delegator,
+ delegate=delegate, readWrite=readWrite)
+
+ def directDelegates(self, delegator, readWrite):
+ return self._selectDelegatesQuery.on(self, delegator=delegator,
+ readWrite=readWrite)
+
+ def groupDelegates(self, delegator, readWrite):
+ return self._selectDelegateGroupssQuery.on(self, delegator=delegator,
+ readWrite=readWrite)
+
+ @inlineCallbacks
+ def delegates(self, delegator, readWrite):
+
+ # First get the direct delegates
+ results = (yield self._selectDelegatesQuery.on(self,
+ delegator=delegator, readWrite=readWrite))
+
+ # Finally get those who are in groups which have been delegated to
+ results.extend((yield self._selectIndirectDelegatesQuery.on(self,
+ delegator=delegator, readWrite=readWrite)))
+
+ returnValue(results)
+
+
+ @inlineCallbacks
+ def delegators(self, delegate, readWrite):
+
+ # First get the direct delegators
+ results = (yield self._selectDirectDelegatorsQuery.on(self,
+ delegate=delegate, readWrite=readWrite))
+
+ # Finally get those who have delegated to groups the delegate
+ # is a member of
+ results.extend((yield self._selectIndirectDelegatorsQuery.on(self,
+ delegate=delegate, readWrite=readWrite)))
+ returnValue(results)
+
+ # End of Delegates
+
+
</ins><span class="cx"> def preCommit(self, operation):
</span><span class="cx"> """
</span><span class="cx"> Run things before C{commit}. (Note: only provided by SQL
</span></span></pre></div>
<a id="CalendarServerbranchesuserssagengroupcachertxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql_schema/current.sql (11890 => 11891)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql_schema/current.sql 2013-11-05 20:05:41 UTC (rev 11890)
+++ CalendarServer/branches/users/sagen/groupcacher/txdav/common/datastore/sql_schema/current.sql 2013-11-06 20:11:38 UTC (rev 11891)
</span><span class="lines">@@ -685,10 +685,9 @@
</span><span class="cx"> GROUP_ID integer
</span><span class="cx"> );
</span><span class="cx">
</span><del>-create sequence GROUP_ID_SEQ;
</del><span class="cx">
</span><span class="cx"> create table GROUPS (
</span><del>- GROUP_ID integer primary key default nextval('GROUP_ID_SEQ'), -- implicit index
</del><ins>+ GROUP_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
</ins><span class="cx"> NAME varchar(255) not null,
</span><span class="cx"> GROUP_GUID varchar(255) not null,
</span><span class="cx"> MEMBERSHIP_HASH varchar(255) not null,
</span><span class="lines">@@ -696,6 +695,7 @@
</span><span class="cx"> CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
</span><span class="cx"> MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
</span><span class="cx"> );
</span><ins>+create index GROUPS_GROUP_GUID on GROUPS(GROUP_GUID);
</ins><span class="cx">
</span><span class="cx"> create table GROUP_MEMBERSHIP (
</span><span class="cx"> GROUP_ID integer,
</span><span class="lines">@@ -710,6 +710,22 @@
</span><span class="cx"> MEMBERSHIP_HASH varchar(255) not null
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+---------------
+-- Delegates --
+---------------
+
+create table DELEGATES (
+ DELEGATOR varchar(255) not null,
+ DELEGATE varchar(255) not null,
+ READ_WRITE integer not null -- 1 = ReadWrite, 0 = ReadOnly
+);
+
+create table DELEGATE_GROUPS (
+ DELEGATOR varchar(255) not null,
+ GROUP_ID integer not null,
+ READ_WRITE integer not null -- 1 = ReadWrite, 0 = ReadOnly
+);
+
</ins><span class="cx"> --------------------------
</span><span class="cx"> -- Object Splitter Work --
</span><span class="cx"> --------------------------
</span></span></pre>
</div>
</div>
</body>
</html>