May 2009 - calendarserver-changes - lists.macosforge.org

[4312] CalendarServer/trunk/contrib/migration/59_calendarmigrator.py
by source_changes＠macosforge.org 28 May '09

28 May '09

Revision: 4312 http://trac.macosforge.org/projects/calendarserver/changeset/4312 Author: sagen(a)apple.com Date: 2009-05-27 18:07:23 -0700 (Wed, 27 May 2009) Log Message: ----------- Rather than just copy old plist over from previous system, merge the keys we care about into the new plist. Modified Paths: -------------- CalendarServer/trunk/contrib/migration/59_calendarmigrator.py Modified: CalendarServer/trunk/contrib/migration/59_calendarmigrator.py =================================================================== --- CalendarServer/trunk/contrib/migration/59_calendarmigrator.py 2009-05-27 22:57:03 UTC (rev 4311) +++ CalendarServer/trunk/contrib/migration/59_calendarmigrator.py 2009-05-28 01:07:23 UTC (rev 4312) @@ -149,16 +149,73 @@ os.symlink(link, newPath) elif os.path.isfile(oldPath): - # Copy the file over, overwriting copy in newConfigDir - log("Copying file %s to %s" % (oldPath, newConfigDir)) - shutil.copy2(oldPath, newConfigDir) + if name == "caldavd.plist": + # Migrate certain settings from the old plist to new: + log("Parsing %s" % (oldPath,)) + oldPlist = plistlib.readPlist(oldPath) + if os.path.exists(newPath): + log("Parsing %s" % (newPath,)) + newPlist = plistlib.readPlist(newPath) + log("Removing %s" % (newPath,)) + os.remove(newPath) + else: + newPlist = { } + log("Processing %s" % (oldPath,)) + mergePlist(oldPlist, newPlist) + log("Writing %s" % (newPath,)) + plistlib.writePlist(newPlist, newPath) + + else: + # Copy the file over, overwriting copy in newConfigDir + log("Copying file %s to %s" % (oldPath, newConfigDir)) + shutil.copy2(oldPath, newConfigDir) + + elif os.path.isdir(oldPath) and not os.path.exists(newPath): # Copy the dir over, but only if new one doesn't exist log("Copying directory %s to %s" % (oldPath, newPath)) shutil.copytree(oldPath, newPath, symlinks=True) +def mergePlist(oldPlist, newPlist): + # The following CalendarServer v1.x keys are ignored: + # EnableNotifications, Verbose + + # These keys are copied verbatim: + for key in ( + "AccessLogFile", "AdminPrincipals", "BindAddresses", "BindHTTPPorts", + "BindSSLPorts", "ControlSocket", "DocumentRoot", "EnableDropBox", + "EnableProxyPrincipals", "EnableSACLs", "ErrorLogFile", "GroupName", + "HTTPPort", "MaximumAttachmentSize", "MultiProcess", "PIDFile", + "ProcessType", "ResponseCompression", "RotateAccessLog", + "SSLAuthorityChain", "SSLCertificate", "SSLPort", "SSLPrivateKey", + "ServerHostName", "ServerStatsFile", "SudoersFile", "UserName", + "UserQuota", + ): + if key in oldPlist: + newPlist[key] = oldPlist[key] + + # "Wiki" is a new authentication in v2.x; copy all "Authentication" sub-keys # over, and "Wiki" will be picked up from the new plist: + if "Authentication" in oldPlist: + for key in oldPlist["Authentication"]: + newPlist["Authentication"][key] = oldPlist["Authentication"][key] + + # Strip out any unknown params from the DirectoryService: + if "DirectoryService" in oldPlist: + newPlist["DirectoryService"] = oldPlist["DirectoryService"] + for key in newPlist["DirectoryService"]["params"].keys(): + if key not in ( + "node", "restrictEnabledRecords", "restrictToGroup", + "cacheTimeout", "xmlFile" + ): + del newPlist["DirectoryService"]["params"][key] + + # Place DataRoot as a sibling of DocumentRoot: + parent = os.path.dirname(newPlist["DocumentRoot"].rstrip("/")) + newPlist["DataRoot"] = os.path.join(parent, "Data") + + def isServiceDisabled(source, service): """ Returns whether or not a service is disabled

1 0

[4311] CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/ accesslog.py
by source_changes＠macosforge.org 27 May '09

27 May '09

Revision: 4311 http://trac.macosforge.org/projects/calendarserver/changeset/4311 Author: wsanchez(a)apple.com Date: 2009-05-27 15:57:03 -0700 (Wed, 27 May 2009) Log Message: ----------- for format str Modified Paths: -------------- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/accesslog.py Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/accesslog.py =================================================================== --- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/accesslog.py 2009-05-27 22:54:26 UTC (rev 4310) +++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/accesslog.py 2009-05-27 22:57:03 UTC (rev 4311) @@ -118,7 +118,7 @@ except AttributeError: serverInstance = "Unknown" - format_str += ' [%d %d]' + format_str += ' [%s %s]' format_data += ( serverInstance, request.chanRequest.channel.factory.outstandingRequests, @@ -133,7 +133,7 @@ self.logDateString(time.time()), ) if config.MoreAccessLogData: - format_str += ' [%d %d]' + format_str += ' [%s %s]' format_data += ( overloaded.transport.server.port, overloaded.outstandingRequests,

1 0

[4310] CalendarServer/trunk/twistedcaldav/accesslog.py
by source_changes＠macosforge.org 27 May '09

27 May '09

Revision: 4310 http://trac.macosforge.org/projects/calendarserver/changeset/4310 Author: wsanchez(a)apple.com Date: 2009-05-27 15:54:26 -0700 (Wed, 27 May 2009) Log Message: ----------- meh Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/accesslog.py Modified: CalendarServer/trunk/twistedcaldav/accesslog.py =================================================================== --- CalendarServer/trunk/twistedcaldav/accesslog.py 2009-05-27 22:52:17 UTC (rev 4309) +++ CalendarServer/trunk/twistedcaldav/accesslog.py 2009-05-27 22:54:26 UTC (rev 4310) @@ -133,7 +133,7 @@ self.logDateString(time.time()), ) if config.MoreAccessLogData: - format_str += ' [%d %d]' + format_str += ' [%s %s]' format_data += ( overloaded.transport.server.port, overloaded.outstandingRequests,

1 0

[4309] CalendarServer/trunk/calendarserver/provision/root.py
by source_changes＠macosforge.org 27 May '09

27 May '09

Revision: 4309 http://trac.macosforge.org/projects/calendarserver/changeset/4309 Author: wsanchez(a)apple.com Date: 2009-05-27 15:52:17 -0700 (Wed, 27 May 2009) Log Message: ----------- comment Modified Paths: -------------- CalendarServer/trunk/calendarserver/provision/root.py Modified: CalendarServer/trunk/calendarserver/provision/root.py =================================================================== --- CalendarServer/trunk/calendarserver/provision/root.py 2009-05-27 22:51:05 UTC (rev 4308) +++ CalendarServer/trunk/calendarserver/provision/root.py 2009-05-27 22:52:17 UTC (rev 4309) @@ -118,7 +118,7 @@ # with an empty string. if authzUser == davxml.Principal(davxml.Unauthenticated()): if RootResource.CheckSACL("", self.saclService) != 0: - log.msg("Unauthenticated users not enabled with the %r SACL" % (self.saclService,)) + log.info("Unauthenticated users not enabled with the %r SACL" % (self.saclService,)) response = (yield UnauthorizedResponse.makeResponse( request.credentialFactories, request.remoteAddr @@ -147,7 +147,7 @@ username = principal.record.shortNames[0] if RootResource.CheckSACL(username, self.saclService) != 0: - log.msg("User %r is not enabled with the %r SACL" % (username, self.saclService,)) + log.info("User %r is not enabled with the %r SACL" % (username, self.saclService,)) raise HTTPError(responsecode.FORBIDDEN) # Mark SACLs as having been checked so we can avoid doing it multiple times @@ -220,6 +220,7 @@ child = (yield super(RootResource, self).locateChild(request, segments)) returnValue(child) + # FIXME: should catch something more specific than Exception except Exception, e: log.warn("Wiki lookup returned ERROR: %s" % (e,)) raise HTTPError(StatusResponse(

1 0

[4308] CalendarServer/trunk/twistedcaldav/accesslog.py
by source_changes＠macosforge.org 27 May '09

27 May '09

Revision: 4308 http://trac.macosforge.org/projects/calendarserver/changeset/4308 Author: wsanchez(a)apple.com Date: 2009-05-27 15:51:05 -0700 (Wed, 27 May 2009) Log Message: ----------- Fix format str Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/accesslog.py Modified: CalendarServer/trunk/twistedcaldav/accesslog.py =================================================================== --- CalendarServer/trunk/twistedcaldav/accesslog.py 2009-05-21 18:26:08 UTC (rev 4307) +++ CalendarServer/trunk/twistedcaldav/accesslog.py 2009-05-27 22:51:05 UTC (rev 4308) @@ -118,7 +118,7 @@ (time.time() - request.initTime) * 1000, ) if config.MoreAccessLogData: - format_str += ' [%d %d]' + format_str += ' [%s %s]' format_data += ( request.serverInstance, request.chanRequest.channel.factory.outstandingRequests,

1 0

[4307] CalendarServer/trunk/twistedcaldav/directory
by source_changes＠macosforge.org 21 May '09

21 May '09

Revision: 4307 http://trac.macosforge.org/projects/calendarserver/changeset/4307 Author: sagen(a)apple.com Date: 2009-05-21 11:26:08 -0700 (Thu, 21 May 2009) Log Message: ----------- If there are duplicate GUIDs, don't fault any of them in (plus unit test) Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py Modified: CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2009-05-20 22:44:57 UTC (rev 4306) +++ CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2009-05-21 18:26:08 UTC (rev 4307) @@ -437,7 +437,8 @@ return deferred - def queryDirectory(self, recordTypes, indexType, indexKey): + def queryDirectory(self, recordTypes, indexType, indexKey, + lookupMethod=opendirectory.queryRecordsWithAttribute_list): attrs = [ dsattributes.kDS1AttrGeneratedUID, @@ -508,7 +509,7 @@ listRecordTypes, attrs, )) - results = opendirectory.queryRecordsWithAttribute_list( + results = lookupMethod( self.directory, query.attribute, query.value, @@ -602,7 +603,7 @@ dsattributes.kDSStdRecordTypeGroups, [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups,], )) - results = opendirectory.queryRecordsWithAttribute_list( + results = lookupMethod( self.directory, attributeToMatch, valueToMatch, @@ -689,8 +690,8 @@ record = disabledRecords[0] elif indexType == self.INDEX_TYPE_GUID and len(enabledRecords) > 1: self.log_error("Duplicate records found for GUID %s:" % (indexKey,)) - for record in enabledRecords: - self.log_error("Duplicate: %s" % (", ".join(record.shortNames))) + for duplicateRecord in enabledRecords: + self.log_error("Duplicate: %s" % (", ".join(duplicateRecord.shortNames))) if record: if isinstance(origIndexKey, unicode): Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py 2009-05-20 22:44:57 UTC (rev 4306) +++ CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py 2009-05-21 18:26:08 UTC (rev 4307) @@ -23,6 +23,7 @@ import twistedcaldav.directory.test.util from twistedcaldav.directory.directory import DirectoryService from twistedcaldav.directory.appleopendirectory import OpenDirectoryRecord + import dsattributes # Wonky hack to prevent unclean reactor shutdowns class DummyReactor(object): @@ -80,3 +81,56 @@ digested = twisted.web2.auth.digest.DigestedCredentials("user", "GET", "example.com", digestFields, None) self.assertFalse(record.verifyCredentials(digested)) + + + def test_queryDirectorySingleGUID(self): + """ Test for lookup on existing and non-existing GUIDs """ + + def lookupMethod(obj, attr, value, matchType, casei, recordType, attributes, count=0): + + data = [ + { + dsattributes.kDS1AttrGeneratedUID : "1234567890", + dsattributes.kDSNAttrRecordName : ["user1", "User 1"], + dsattributes.kDSNAttrRecordType : dsattributes.kDSStdRecordTypeUsers, + }, + ] + results = [] + for entry in data: + if entry[attr] == value: + results.append(("", entry)) + return results + + recordTypes = [DirectoryService.recordType_users, DirectoryService.recordType_groups, DirectoryService.recordType_locations, DirectoryService.recordType_resources] + self.service().queryDirectory(recordTypes, self.service().INDEX_TYPE_GUID, "1234567890", lookupMethod=lookupMethod) + self.assertTrue(self.service().recordWithGUID("1234567890")) + self.assertFalse(self.service().recordWithGUID("987654321")) + + + def test_queryDirectoryDuplicateGUIDs(self): + """ Test for lookup on duplicate GUIDs, ensuring they don't get + faulted in """ + + def lookupMethod(obj, attr, value, matchType, casei, recordType, attributes, count=0): + + data = [ + { + dsattributes.kDS1AttrGeneratedUID : "1234567890", + dsattributes.kDSNAttrRecordName : ["user1", "User 1"], + dsattributes.kDSNAttrRecordType : dsattributes.kDSStdRecordTypeUsers, + }, + { + dsattributes.kDS1AttrGeneratedUID : "1234567890", + dsattributes.kDSNAttrRecordName : ["user2", "User 2"], + dsattributes.kDSNAttrRecordType : dsattributes.kDSStdRecordTypeUsers, + }, + ] + results = [] + for entry in data: + if entry[attr] == value: + results.append(("", entry)) + return results + + recordTypes = [DirectoryService.recordType_users, DirectoryService.recordType_groups, DirectoryService.recordType_locations, DirectoryService.recordType_resources] + self.service().queryDirectory(recordTypes, self.service().INDEX_TYPE_GUID, "1234567890", lookupMethod=lookupMethod) + self.assertFalse(self.service().recordWithGUID("1234567890"))

1 0

[4306] CalendarServer/trunk/contrib/migration/59_calendarmigrator.py
by source_changes＠macosforge.org 20 May '09

20 May '09

Revision: 4306 http://trac.macosforge.org/projects/calendarserver/changeset/4306 Author: sagen(a)apple.com Date: 2009-05-20 15:44:57 -0700 (Wed, 20 May 2009) Log Message: ----------- Copy config, *then* start server Modified Paths: -------------- CalendarServer/trunk/contrib/migration/59_calendarmigrator.py Modified: CalendarServer/trunk/contrib/migration/59_calendarmigrator.py =================================================================== --- CalendarServer/trunk/contrib/migration/59_calendarmigrator.py 2009-05-20 17:44:08 UTC (rev 4305) +++ CalendarServer/trunk/contrib/migration/59_calendarmigrator.py 2009-05-20 22:44:57 UTC (rev 4306) @@ -70,8 +70,8 @@ if options.sourceRoot: if os.path.exists(options.sourceRoot): + migrateConfiguration(options) migrateRunState(options) - migrateConfiguration(options) else: log("ERROR: --sourceRoot must be specified")

1 0

[4305] CalendarServer/trunk/calendarserver/tools/principals.py
by source_changes＠macosforge.org 20 May '09

20 May '09

Revision: 4305 http://trac.macosforge.org/projects/calendarserver/changeset/4305 Author: sagen(a)apple.com Date: 2009-05-20 10:44:08 -0700 (Wed, 20 May 2009) Log Message: ----------- Set umask so db files get created with appropriate permissions Modified Paths: -------------- CalendarServer/trunk/calendarserver/tools/principals.py Modified: CalendarServer/trunk/calendarserver/tools/principals.py =================================================================== --- CalendarServer/trunk/calendarserver/tools/principals.py 2009-05-20 16:23:22 UTC (rev 4304) +++ CalendarServer/trunk/calendarserver/tools/principals.py 2009-05-20 17:44:08 UTC (rev 4305) @@ -200,6 +200,8 @@ gid = getgrnam(config.GroupName).gr_gid switchUID(uid, uid, gid) + os.umask(config.umask) + config.directory = getDirectory() autoDisableMemcached(config) except ConfigurationError, e:

1 0

[4304] CalendarServer/trunk/memcacheclient.py
by source_changes＠macosforge.org 20 May '09

20 May '09

Revision: 4304 http://trac.macosforge.org/projects/calendarserver/changeset/4304 Author: sagen(a)apple.com Date: 2009-05-20 09:23:22 -0700 (Wed, 20 May 2009) Log Message: ----------- This check-and-set message is not an error -- dropping to log.debug Modified Paths: -------------- CalendarServer/trunk/memcacheclient.py Modified: CalendarServer/trunk/memcacheclient.py =================================================================== --- CalendarServer/trunk/memcacheclient.py 2009-05-20 03:52:53 UTC (rev 4303) +++ CalendarServer/trunk/memcacheclient.py 2009-05-20 16:23:22 UTC (rev 4304) @@ -739,7 +739,7 @@ raise NotFoundError(key) if token and result == "EXISTS": - log.error("Memcacheclient check-and-set failed") + log.debug("Memcacheclient check-and-set failed") raise TokenMismatchError(key) log.error("Memcacheclient %s command failed with result (%s)" %

1 0

[4303] CalendarServer/trunk/twistedcaldav/index.py
by source_changes＠macosforge.org 20 May '09

20 May '09

Revision: 4303 http://trac.macosforge.org/projects/calendarserver/changeset/4303 Author: cdaboo(a)apple.com Date: 2009-05-19 20:52:53 -0700 (Tue, 19 May 2009) Log Message: ----------- Add some extra logging for failed instance expansion when indexing. Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/index.py Modified: CalendarServer/trunk/twistedcaldav/index.py =================================================================== --- CalendarServer/trunk/twistedcaldav/index.py 2009-05-20 00:41:58 UTC (rev 4302) +++ CalendarServer/trunk/twistedcaldav/index.py 2009-05-20 03:52:53 UTC (rev 4303) @@ -49,6 +49,7 @@ from twistedcaldav.sql import AbstractSQLDatabase from twistedcaldav.sql import db_prefix from twistedcaldav import caldavxml +from twistedcaldav.instance import InvalidOverriddenInstanceError from twistedcaldav.log import Logger, LoggingMixIn from twistedcaldav.config import config from twistedcaldav.memcachepool import CachePoolUserMixIn @@ -486,7 +487,11 @@ if expand > (datetime.date.today() + maximum_future_expansion_duration): raise IndexedSearchException - instances = calendar.expandTimeRanges(expand) + try: + instances = calendar.expandTimeRanges(expand) + except InvalidOverriddenInstanceError, e: + log.err("Invalid instance %s when indexing %s in %s" % (e.rid, name, self.resource,)) + raise self._delete_from_db(name, uid)

1 0