[CalendarServer-changes] [7771] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Mon Jul 11 15:10:07 PDT 2011
Revision: 7771
http://trac.macosforge.org/projects/calendarserver/changeset/7771
Author: sagen at apple.com
Date: 2011-07-11 15:10:07 -0700 (Mon, 11 Jul 2011)
Log Message:
-----------
Removed the indicator that the group membership cache was populated; instead the master now populates it prior to the workers starting. Also the most recent
membership data is stored in a file for quick loading after a restart.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/directory/directory.py
CalendarServer/trunk/twistedcaldav/directory/principal.py
CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Modified: CalendarServer/trunk/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/directory.py 2011-07-11 22:08:29 UTC (rev 7770)
+++ CalendarServer/trunk/twistedcaldav/directory/directory.py 2011-07-11 22:10:07 UTC (rev 7771)
@@ -28,15 +28,19 @@
"UnknownRecordTypeError",
]
+import os
import sys
import types
+import pwd, grp
+import cPickle as pickle
+
from zope.interface import implements
from twisted.cred.error import UnauthorizedLogin
from twisted.cred.checkers import ICredentialsChecker
from twext.web2.dav.auth import IPrincipalCredentials
-from twisted.internet.defer import succeed, inlineCallbacks
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from twext.python.log import LoggingMixIn
@@ -47,6 +51,7 @@
from twistedcaldav import servers
from twistedcaldav.memcacher import Memcacher
from twistedcaldav import memcachepool
+from twisted.python.filepath import FilePath
from twisted.python.reflect import namedClass
from twisted.python.usage import Options, UsageError
from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
@@ -349,7 +354,7 @@
raise NotImplementedError("Subclass must implement createRecords")
@inlineCallbacks
- def cacheGroupMembership(self, guids):
+ def cacheGroupMembership(self, guids, fast=False):
"""
Update the "which groups is each principal in" cache. The only groups
that the server needs to worry about are the ones which have been
@@ -360,26 +365,60 @@
guids is the set of every guid that's been directly delegated to, and
can be a mixture of users and groups.
"""
- groups = set()
- for guid in guids:
- record = self.recordWithGUID(guid)
- if record is not None and record.recordType == self.recordType_groups:
- groups.add(record)
- members = { }
- for group in groups:
- groupMembers = group.expandedMembers()
- for member in groupMembers:
- if member.recordType == self.recordType_users:
- memberships = members.setdefault(member.guid, set())
- memberships.add(group.guid)
+ dataRoot = FilePath(config.DataRoot)
+ snapshotFile = dataRoot.child("memberships_cache")
+ if not snapshotFile.exists():
+ fast = False
+
+ if fast:
+ # If there is an on-disk snapshot of the membership information,
+ # load that and put into memcached, bypassing the faulting in of
+ # any records, so that the server can start up quickly.
+
+ self.log_debug("Loading group memberships from snapshot")
+ members = pickle.loads(snapshotFile.getContent())
+
+ else:
+ self.log_debug("Loading group memberships from directory")
+ groups = set()
+ for guid in guids:
+ record = self.recordWithGUID(guid)
+ if record is not None and record.recordType == self.recordType_groups:
+ groups.add(record)
+
+ members = { }
+ for group in groups:
+ groupMembers = group.expandedMembers()
+ for member in groupMembers:
+ if member.recordType == self.recordType_users:
+ memberships = members.setdefault(member.guid, set())
+ memberships.add(group.guid)
+
+ # Store snapshot
+ self.log_debug("Taking snapshot of group memberships to %s" %
+ (snapshotFile.path,))
+ snapshotFile.setContent(pickle.dumps(members))
+
+ # Update ownership
+ uid = gid = -1
+ if config.UserName:
+ uid = pwd.getpwname(config.UserName).pw_uid
+ if config.GroupName:
+ gid = grp.getgrname(config.GroupName).gr_gid
+ os.chown(snapshotFile.path, uid, gid)
+
+ self.log_debug("Storing group memberships in memcached")
for member, groups in members.iteritems():
+ # self.log_debug("%s is in %s" % (member, groups))
yield self.groupMembershipCache.setGroupsFor(member, groups)
- self.groupMembershipCache.createMarker()
+ self.log_debug("Group memberships cache updated")
+ returnValue((fast, len(members)))
+
class GroupMembershipCache(Memcacher, LoggingMixIn):
"""
Caches group membership information
@@ -425,21 +464,8 @@
d.addCallback(_value)
return d
- def deleteGroupsFor(self, guid, proxyType):
- return self.delete("groups-for:%s" % (str(guid),))
- def createMarker(self):
- return self.set("proxy-cache-populated", "true",
- expire_time=self.expireSeconds)
- def checkMarker(self):
- def _value(value):
- return value == "true"
- d = self.get("proxy-cache-populated")
- d.addCallback(_value)
- return d
-
-
class GroupMembershipCacheUpdater(LoggingMixIn):
"""
Responsible for updating memcached with group memberships. This will run
@@ -459,7 +485,7 @@
self.cache = cache
@inlineCallbacks
- def updateCache(self):
+ def updateCache(self, fast=False):
"""
Iterate the proxy database to retrieve all the principals who have been
delegated to. Fault these principals in. For any of these principals
@@ -482,8 +508,8 @@
for guid in (yield self.proxyDB.getMembers(proxyGroup)):
guids.add(guid)
+ returnValue((yield self.directory.cacheGroupMembership(guids, fast=fast)))
- yield self.directory.cacheGroupMembership(guids)
class GroupMembershipCacherOptions(Options):
@@ -616,6 +642,7 @@
self.nextUpdate.cancel()
+
class GroupMembershipCacherServiceMaker(LoggingMixIn):
"""
Configures and returns a GroupMembershipCacherService
@@ -658,7 +685,6 @@
return cacherService
-
class DirectoryRecord(LoggingMixIn):
implements(IDirectoryRecord)
Modified: CalendarServer/trunk/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/principal.py 2011-07-11 22:08:29 UTC (rev 7770)
+++ CalendarServer/trunk/twistedcaldav/directory/principal.py 2011-07-11 22:10:07 UTC (rev 7771)
@@ -41,7 +41,7 @@
from twisted.internet.defer import succeed
from twext.web2.auth.digest import DigestedCredentials
from twext.web2 import responsecode
-from twext.web2.http import HTTPError, StatusResponse
+from twext.web2.http import HTTPError
from twext.web2.dav import davxml
from twext.web2.dav.util import joinURL
from twext.web2.dav.noneprops import NonePropertyStore
@@ -670,9 +670,6 @@
cache = getattr(self.record.service, "groupMembershipCache", None)
if cache:
log.debug("proxyFor is using groupMembershipCache")
- if not (yield cache.checkMarker()):
- raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
- "Group membership cache not yet populated"))
guids = (yield self.record.cachedGroups())
memberships = set()
for guid in guids:
@@ -750,9 +747,6 @@
cache = getattr(self.record.service, "groupMembershipCache", None)
if cache:
log.debug("groupMemberships is using groupMembershipCache")
- if not (yield cache.checkMarker()):
- raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
- "Group membership cache not yet populated"))
guids = (yield self.record.cachedGroups())
groups = set()
for guid in guids:
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py 2011-07-11 22:08:29 UTC (rev 7770)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py 2011-07-11 22:10:07 UTC (rev 7771)
@@ -16,7 +16,7 @@
from twisted.internet.defer import inlineCallbacks
from twisted.internet.task import Clock
-from twext.web2.http import HTTPError
+from twisted.python.filepath import FilePath
from twistedcaldav.test.util import TestCase
from twistedcaldav.test.util import xmlFile, augmentsFile, proxiesFile
@@ -26,8 +26,9 @@
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
from twistedcaldav.directory import augment, calendaruserproxy
from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
-
+import cPickle as pickle
+
def StubCheckSACL(cls, username, service):
services = {
"calendar" : ["amanda", "betty"],
@@ -136,34 +137,6 @@
- @inlineCallbacks
- def test_groupMembershipCacheMarker(self):
- """
- If the group member cache is not populated (as noted by the existence
- of a special memcached key), a 503 should be raised
- """
- cache = GroupMembershipCache("ProxyDB")
- # Having a groupMembershipCache assigned to the directory service is the
- # trigger to use such a cache:
- self.directoryService.groupMembershipCache = cache
-
- userc = self._getPrincipalByShortName(DirectoryService.recordType_users, "userc")
-
- try:
- yield userc.proxyFor(True)
- except HTTPError:
- pass
- else:
- self.fail("HTTPError was unexpectedly not raised")
-
- try:
- yield userc.groupMemberships(True)
- except HTTPError:
- pass
- else:
- self.fail("HTTPError was unexpectedly not raised")
-
-
def test_expandedMembers(self):
"""
Make sure expandedMembers( ) returns a complete, flattened set of
@@ -298,3 +271,79 @@
set(uids),
groups,
)
+
+
+ @inlineCallbacks
+ def test_groupMembershipCacheSnapshot(self):
+ """
+ The group membership cache creates a snapshot (a pickle file) of
+ the member -> groups dictionary, and can quickly refresh memcached
+ from that snapshot when restarting the server.
+ """
+ cache = GroupMembershipCache("ProxyDB", 60)
+ # Having a groupMembershipCache assigned to the directory service is the
+ # trigger to use such a cache:
+ self.directoryService.groupMembershipCache = cache
+
+ updater = GroupMembershipCacheUpdater(
+ calendaruserproxy.ProxyDBService, self.directoryService, 30,
+ cache=cache)
+
+ dataRoot = FilePath(config.DataRoot)
+ snapshotFile = dataRoot.child("memberships_cache")
+
+ # Snapshot doesn't exist initially
+ self.assertFalse(snapshotFile.exists())
+
+ # Try a fast update (as when the server starts up for the very first
+ # time), but since the snapshot doesn't exist we fault in from the
+ # directory (fast now is False), and snapshot will get created
+ fast, numMembers = (yield updater.updateCache(fast=True))
+ self.assertEquals(fast, False)
+ self.assertEquals(numMembers, 4)
+ self.assertTrue(snapshotFile.exists())
+
+ # Try another fast update where the snapshot already exists (as in a
+ # server-restart scenario), which will only read from the snapshot
+ # as indicated by the return value for "fast"
+ fast, numMembers = (yield updater.updateCache(fast=True))
+ self.assertEquals(fast, True)
+ self.assertEquals(numMembers, 4)
+
+ # Try an update which faults in from the directory (fast=False)
+ fast, numMembers = (yield updater.updateCache(fast=False))
+ self.assertEquals(fast, False)
+ self.assertEquals(numMembers, 4)
+
+ # Verify the snapshot contains the pickled dictionary we expect
+ members = pickle.loads(snapshotFile.getContent())
+ self.assertEquals(
+ members,
+ {
+ "5A985493-EE2C-4665-94CF-4DFEA3A89500":
+ set([
+ "non_calendar_group",
+ "recursive1_coasts",
+ "recursive2_coasts",
+ "both_coasts"
+ ]),
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D":
+ set([
+ "left_coast",
+ "recursive1_coasts",
+ "recursive2_coasts",
+ "both_coasts"
+ ]),
+ "5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1":
+ set([
+ "left_coast",
+ "both_coasts"
+ ]),
+ "8B4288F6-CC82-491D-8EF9-642EF4F3E7D0":
+ set([
+ "non_calendar_group",
+ "left_coast",
+ "both_coasts"
+ ])
+ }
+ )
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2011-07-11 22:08:29 UTC (rev 7770)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2011-07-11 22:10:07 UTC (rev 7771)
@@ -24,11 +24,14 @@
from twext.web2.dav.fileop import rmdir
from twext.web2.dav import davxml
from twext.python.log import Logger
+from twisted.python.reflect import namedClass
+
from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
from twistedcaldav.directory.xmlfile import XMLDirectoryService
from twistedcaldav.directory.calendaruserproxy import ProxySqliteDB
-from twistedcaldav.directory.directory import DirectoryService
+from twistedcaldav.directory.directory import DirectoryService, GroupMembershipCacheUpdater
+from twistedcaldav.directory import calendaruserproxy
from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
from twistedcaldav.mail import MailGatewayTokensDatabase
from twistedcaldav.ical import Component
@@ -44,7 +47,7 @@
from txdav.caldav.datastore.index_file import db_basename
-from calendarserver.tap.util import getRootResource, FakeRequest
+from calendarserver.tap.util import getRootResource, FakeRequest, directoryFromConfig
from calendarserver.tools.util import getDirectory
from calendarserver.tools.resources import migrateResources
@@ -809,8 +812,13 @@
class PostDBImportService(Service, object):
"""
- Service for processing non-implicit inbox items after data has been
- imported into the DB
+ Service which runs after database import but before workers are spawned
+ (except memcached will be running at this point)
+
+ The jobs carried out here are:
+
+ 1. Populating the group-membership cache
+ 2. Processing non-implicit inbox items
"""
def __init__(self, config, store, service):
@@ -821,13 +829,30 @@
self.store = store
self.config = config
+ @inlineCallbacks
def startService(self):
"""
Start the service.
"""
- self.processInboxItems()
+ # Populate the group membership cache
+ if (self.config.GroupCaching.Enabled and
+ self.config.GroupCaching.EnableUpdater):
+ proxydb = calendaruserproxy.ProxyDBService
+ if proxydb is None:
+ proxydbClass = namedClass(self.config.ProxyDBService.type)
+ proxydb = proxydbClass(**self.config.ProxyDBService.params)
+ directory = directoryFromConfig(self.config)
+ updater = GroupMembershipCacheUpdater(proxydb,
+ directory, self.config.GroupCaching.ExpireSeconds,
+ namespace=self.config.GroupCaching.MemcachedPool)
+ yield updater.updateCache(fast=True)
+
+ # Process old inbox items
+ yield self.processInboxItems()
+
+
@inlineCallbacks
def processInboxItems(self):
"""
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110711/f5d610fc/attachment-0001.html>
More information about the calendarserver-changes
mailing list