January 2009 - calendarserver-changes

[3575] CalDAVTester/branches/users/cdaboo/location-partial-accept-3574/
by source_changes＠macosforge.org 07 Jan '09

07 Jan '09

Revision: 3575 http://trac.macosforge.org/projects/calendarserver/changeset/3575 Author: cdaboo(a)apple.com Date: 2009-01-07 12:19:35 -0800 (Wed, 07 Jan 2009) Log Message: ----------- Added Paths: ----------- CalDAVTester/branches/users/cdaboo/location-partial-accept-3574/

1 0

[3574] CalendarServer/branches/users/cdaboo/location-partial-accept-3573/
by source_changes＠macosforge.org 07 Jan '09

07 Jan '09

Revision: 3574 http://trac.macosforge.org/projects/calendarserver/changeset/3574 Author: cdaboo(a)apple.com Date: 2009-01-07 12:18:47 -0800 (Wed, 07 Jan 2009) Log Message: ----------- Branch to work on having locations partially accept recurring items. Added Paths: ----------- CalendarServer/branches/users/cdaboo/location-partial-accept-3573/

1 0

[3573] CalendarServer/trunk/twistedcaldav/directory
by source_changes＠macosforge.org 07 Jan '09

07 Jan '09

Revision: 3573 http://trac.macosforge.org/projects/calendarserver/changeset/3573 Author: cdaboo(a)apple.com Date: 2009-01-07 10:15:56 -0800 (Wed, 07 Jan 2009) Log Message: ----------- Delete entries from the delegate DB if users are no longer in the directory. Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py CalendarServer/trunk/twistedcaldav/directory/principal.py CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2009-01-06 20:23:10 UTC (rev 3572) +++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2009-01-07 18:15:56 UTC (rev 3573) @@ -1,5 +1,5 @@ ## -# Copyright (c) 2006-2008 Apple Inc. All rights reserved. +# Copyright (c) 2006-2009 Apple Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -70,7 +70,7 @@ return davxml.ACL(*aces) def accessControlList(self, request, inheritance=True, expanding=False, inherited_aces=None): - # Permissions here are fixed, and are not subject to inherritance rules, etc. + # Permissions here are fixed, and are not subject to inheritance rules, etc. return succeed(self.defaultAccessControlList()) class CalendarUserProxyPrincipalResource (CalDAVComplianceMixIn, PermissionsMixIn, DAVPrincipalResource, DAVFile): @@ -196,12 +196,15 @@ principals.append(principal) yield principal.cacheNotifier.changed() + yield self.setGroupMemberSetPrincipals(principals) + yield self.parent.cacheNotifier.changed() + returnValue(True) + + @inlineCallbacks + def setGroupMemberSetPrincipals(self, principals): # Map the principals to UIDs. uids = [p.principalUID() for p in principals] - yield self._index().setGroupMembers(self.uid, uids) - yield self.parent.cacheNotifier.changed() - returnValue(True) ## # HTTP @@ -314,7 +317,20 @@ if self.hasEditableMembership(): # Get member UIDs from database and map to principal resources members = yield self._index().getMembers(self.uid) - returnValue([p for p in [self.pcollection.principalForUID(uid) for uid in members] if p]) + found = [] + missing = [] + for uid in members: + p = self.pcollection.principalForUID(uid) + if p: + found.append(p) + else: + missing.append(uid) + + # Clean-up ones that are missing + for uid in missing: + yield self._index().removePrincipal(uid) + + returnValue(found) else: # Fixed proxies if self.proxyType == "calendar-proxy-write": @@ -357,10 +373,10 @@ class ProxyDBMemcacher(Memcacher): def setMembers(self, guid, members): - return self.set("members:%s" % (guid,), str(",".join(members))) + return self.set("members:%s" % (str(guid),), str(",".join(members))) def setMemberships(self, guid, memberships): - return self.set("memberships:%s" % (guid,), str(",".join(memberships))) + return self.set("memberships:%s" % (str(guid),), str(",".join(memberships))) def getMembers(self, guid): def _value(value): @@ -370,7 +386,7 @@ return None else: return set() - d = self.get("members:%s" % (guid,)) + d = self.get("members:%s" % (str(guid),)) d.addCallback(_value) return d @@ -382,15 +398,15 @@ return None else: return set() - d = self.get("memberships:%s" % (guid,)) + d = self.get("memberships:%s" % (str(guid),)) d.addCallback(_value) return d def deleteMember(self, guid): - return self.delete("members:%s" % (guid,)) + return self.delete("members:%s" % (str(guid),)) def deleteMembership(self, guid): - return self.delete("memberships:%s" % (guid,)) + return self.delete("memberships:%s" % (str(guid),)) def __init__(self, path): path = os.path.join(path, CalendarUserProxyDatabase.dbFilename) @@ -435,17 +451,46 @@ @param principalUID: the UID of the group principal to remove. """ + # Need to get the members before we do the delete + members = yield self.getMembers(principalUID) + self._delete_from_db(principalUID) self._db_commit() # Update cache - members = yield self.getMembers(principalUID) if members: for member in members: yield self._memcacher.deleteMembership(member) yield self._memcacher.deleteMember(principalUID) @inlineCallbacks + def removePrincipal(self, principalUID): + """ + Remove a group membership record. + + @param principalUID: the UID of the principal to remove. + """ + + for suffix in ("calendar-proxy-read", "calendar-proxy-write",): + groupUID = "%s#%s" % (principalUID, suffix,) + self._delete_from_db(groupUID) + + # Update cache + members = yield self.getMembers(groupUID) + if members: + for member in members: + yield self._memcacher.deleteMembership(member) + yield self._memcacher.deleteMember(groupUID) + + memberships = (yield self.getMemberships(principalUID)) + for groupUID in memberships: + yield self._memcacher.deleteMember(groupUID) + + self._delete_from_db_member(principalUID) + yield self._memcacher.deleteMembership(principalUID) + self._db_commit() + + @inlineCallbacks def getMembers(self, principalUID): """ Return the list of group member UIDs for the specified principal. @@ -454,10 +499,7 @@ """ def _members(): - members = set() - for row in self._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", principalUID): - members.add(row[0]) - return members + return set([row[0] for row in self._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", principalUID)]) # Pull from cache result = yield self._memcacher.getMembers(principalUID) @@ -475,10 +517,7 @@ """ def _members(): - members = set() - for row in self._db_execute("select GROUPNAME from GROUPS where MEMBER = :1", principalUID): - members.add(row[0]) - return members + return set([row[0] for row in self._db_execute("select GROUPNAME from GROUPS where MEMBER = :1", principalUID)]) # Pull from cache result = yield self._memcacher.getMemberships(principalUID) @@ -510,6 +549,14 @@ """ self._db_execute("delete from GROUPS where GROUPNAME = :1", principalUID) + def _delete_from_db_member(self, principalUID): + """ + Deletes the specified member entry from the database. + + @param principalUID: the UID of the member principal to remove. + """ + self._db_execute("delete from GROUPS where MEMBER = :1", principalUID) + def _db_version(self): """ @return: the schema version assigned to this index. Modified: CalendarServer/trunk/twistedcaldav/directory/principal.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/principal.py 2009-01-06 20:23:10 UTC (rev 3572) +++ CalendarServer/trunk/twistedcaldav/directory/principal.py 2009-01-07 18:15:56 UTC (rev 3573) @@ -1,5 +1,5 @@ ## -# Copyright (c) 2006-2007 Apple Inc. All rights reserved. +# Copyright (c) 2006-2009 Apple Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -645,6 +645,8 @@ subprincipal = self.parent.principalForUID(uid) if subprincipal: proxies.append(subprincipal) + else: + yield self._calendar_user_proxy_index().removeGroup(uid) groups.update(proxies) @@ -677,8 +679,11 @@ memberships = (yield self._calendar_user_proxy_index().getMemberships(self.principalUID())) for uid in memberships: subprincipal = self.parent.principalForUID(uid) - if subprincipal and subprincipal.isProxyType(read_write): - proxies.append(subprincipal.parent) + if subprincipal: + if subprincipal.isProxyType(read_write): + proxies.append(subprincipal.parent) + else: + yield self._calendar_user_proxy_index().removeGroup(uid) proxyFors.update(proxies) Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py 2009-01-06 20:23:10 UTC (rev 3572) +++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py 2009-01-07 18:15:56 UTC (rev 3573) @@ -1,5 +1,5 @@ ## -# Copyright (c) 2005-2007 Apple Inc. All rights reserved. +# Copyright (c) 2005-2009 Apple Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -270,6 +270,82 @@ self.assertEqual(membershipsD, set()) @inlineCallbacks + def test_cachingDBRemoveSpecial(self): + + for processType in ("Single", "Combined",): + config.ProcessType = processType + + # Get the DB + db_path = self.mktemp() + os.mkdir(db_path) + db = CalendarUserProxyDatabase(db_path) + + # Do one insert and check the result + yield db.setGroupMembers("A", ("B", "C", "D",)) + yield db.setGroupMembers("X", ("B", "C",)) + + membershipsB = yield db.getMemberships("B") + membershipsC = yield db.getMemberships("C") + membershipsD = yield db.getMemberships("D") + + # Remove and check the result + yield db.removeGroup("A") + + membersA = yield db.getMembers("A") + membersX = yield db.getMembers("X") + membershipsB = yield db.getMemberships("B") + membershipsC = yield db.getMemberships("C") + membershipsD = yield db.getMemberships("D") + + self.assertEqual(membersA, set()) + self.assertEqual(membersX, set(("B", "C",))) + self.assertEqual(membershipsB, set("X",)) + self.assertEqual(membershipsC, set("X",)) + self.assertEqual(membershipsD, set()) + + @inlineCallbacks + def test_cachingDBRemovePrincipal(self): + + for processType in ("Single", "Combined",): + config.ProcessType = processType + + # Get the DB + db_path = self.mktemp() + os.mkdir(db_path) + db = CalendarUserProxyDatabase(db_path) + + # Do one insert and check the result + yield db.setGroupMembers("A", ("B", "C", "D",)) + yield db.setGroupMembers("X", ("B", "C",)) + + membersA = yield db.getMembers("A") + membersX = yield db.getMembers("X") + membershipsB = yield db.getMemberships("B") + membershipsC = yield db.getMemberships("C") + membershipsD = yield db.getMemberships("D") + + self.assertEqual(membersA, set(("B", "C", "D",))) + self.assertEqual(membersX, set(("B", "C",))) + self.assertEqual(membershipsB, set(("A", "X",))) + self.assertEqual(membershipsC, set(("A", "X",))) + self.assertEqual(membershipsD, set(("A",))) + + # Remove and check the result + yield db.removePrincipal("B") + + membersA = yield db.getMembers("A") + membersX = yield db.getMembers("X") + membershipsB = yield db.getMemberships("B") + membershipsC = yield db.getMemberships("C") + membershipsD = yield db.getMemberships("D") + + self.assertEqual(membersA, set(("C", "D",))) + self.assertEqual(membersX, set(("C",))) + self.assertEqual(membershipsB, set()) + self.assertEqual(membershipsC, set(("A", "X",))) + self.assertEqual(membershipsD, set(("A",),)) + + @inlineCallbacks def test_cachingDBInsertUncached(self): for processType in ("Single", "Combined",): Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-01-06 20:23:10 UTC (rev 3572) +++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-01-07 18:15:56 UTC (rev 3573) @@ -1,5 +1,5 @@ ## -# Copyright (c) 2005-2007 Apple Inc. All rights reserved. +# Copyright (c) 2005-2009 Apple Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,10 +18,11 @@ from twisted.web2.dav import davxml from twistedcaldav.directory.directory import DirectoryService -from twistedcaldav.directory.xmlfile import XMLDirectoryService from twistedcaldav.directory.test.test_xmlfile import xmlFile from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource from twistedcaldav.directory.principal import DirectoryPrincipalResource +from twistedcaldav.directory.xmlaccountsparser import XMLAccountsParser +from twistedcaldav.directory.xmlfile import XMLDirectoryService import twistedcaldav.test.util @@ -76,6 +77,42 @@ return d @inlineCallbacks + def _addProxy(self, principal, subPrincipalName, proxyPrincipal): + + if isinstance(principal, tuple): + principal = self._getPrincipalByShortName(principal[0], principal[1]) + principal = principal.getChild(subPrincipalName) + members = (yield principal.groupMembers()) + + if isinstance(proxyPrincipal, tuple): + proxyPrincipal = self._getPrincipalByShortName(proxyPrincipal[0], proxyPrincipal[1]) + members.add(proxyPrincipal) + + principal.setGroupMemberSetPrincipals(members) + + @inlineCallbacks + def _removeProxy(self, recordType, recordName, subPrincipalName, proxyRecordType, proxyRecordName): + + principal = self._getPrincipalByShortName(recordType, recordName) + principal = principal.getChild(subPrincipalName) + members = (yield principal.groupMembers()) + + proxyPrincipal = self._getPrincipalByShortName(proxyRecordType, proxyRecordName) + for p in members: + if p.principalUID() == proxyPrincipal.principalUID(): + members.remove(p) + break + + principal.setGroupMemberSetPrincipals(members) + + def _clearProxy(self, principal, subPrincipalName): + + if isinstance(principal, tuple): + principal = self._getPrincipalByShortName(principal[0], principal[1]) + principal = principal.getChild(subPrincipalName) + principal.setGroupMemberSetPrincipals(set()) + + @inlineCallbacks def _proxyForTest(self, recordType, recordName, expectedProxies, read_write): principal = self._getPrincipalByShortName(recordType, recordName) proxies = (yield principal.proxyFor(read_write)) @@ -278,3 +315,121 @@ False ) + @inlineCallbacks + def test_UserProxy(self): + + for proxyType in ("calendar-proxy-read", "calendar-proxy-write"): + + yield self._addProxy( + (DirectoryService.recordType_users, "wsanchez",), + proxyType, + (DirectoryService.recordType_users, "cdaboo",), + ) + + yield self._groupMembersTest( + DirectoryService.recordType_users, "wsanchez", + proxyType, + ("Cyrus Daboo",), + ) + + yield self._addProxy( + (DirectoryService.recordType_users, "wsanchez",), + proxyType, + (DirectoryService.recordType_users, "lecroy",), + ) + + yield self._groupMembersTest( + DirectoryService.recordType_users, "wsanchez", + proxyType, + ("Cyrus Daboo", "Chris Lecroy",), + ) + + yield self._removeProxy( + DirectoryService.recordType_users, "wsanchez", + proxyType, + DirectoryService.recordType_users, "cdaboo", + ) + + yield self._groupMembersTest( + DirectoryService.recordType_users, "wsanchez", + proxyType, + ("Chris Lecroy",), + ) + + @inlineCallbacks + def test_InvalidUserProxy(self): + + for doMembershipFirst in (True, False,): + for proxyType in ("calendar-proxy-read", "calendar-proxy-write"): + principal = self._getPrincipalByShortName(DirectoryService.recordType_users, "wsanchez") + proxyGroup = principal.getChild(proxyType) + + testPrincipal = self._getPrincipalByShortName(DirectoryService.recordType_users, "cdaboo") + + fakePrincipal = self._getPrincipalByShortName(DirectoryService.recordType_users, "dreid") + fakeProxyGroup = fakePrincipal.getChild(proxyType) + + yield self._addProxy( + principal, + proxyType, + testPrincipal, + ) + members = yield proxyGroup._index().getMembers(proxyGroup.uid) + self.assertEquals(len(members), 1) + + yield self._addProxy( + fakePrincipal, + proxyType, + testPrincipal, + ) + members = yield fakeProxyGroup._index().getMembers(fakeProxyGroup.uid) + self.assertEquals(len(members), 1) + + uids = [p.principalUID() for p in (yield testPrincipal.groupMemberships())] + self.assertTrue("5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1#%s" % (proxyType,) in uids) + + memberships = yield testPrincipal._calendar_user_proxy_index().getMemberships(testPrincipal.principalUID()) + self.assertEquals(len(memberships), 2) + + yield self._addProxy( + principal, + proxyType, + fakePrincipal, + ) + members = yield proxyGroup._index().getMembers(proxyGroup.uid) + self.assertEquals(len(members), 2) + + # Remove the dreid user from the directory service + del directoryService._accounts()[DirectoryService.recordType_users]["dreid"] + + @inlineCallbacks + def _membershipTest(): + uids = [p.principalUID() for p in (yield testPrincipal.groupMemberships())] + self.assertTrue("5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1#%s" % (proxyType,) not in uids) + + memberships = yield testPrincipal._calendar_user_proxy_index().getMemberships(testPrincipal.principalUID()) + self.assertEquals(len(memberships), 1) + + @inlineCallbacks + def _membersTest(): + yield self._groupMembersTest( + DirectoryService.recordType_users, "wsanchez", + proxyType, + ("Cyrus Daboo",), + ) + members = yield proxyGroup._index().getMembers(proxyGroup.uid) + self.assertEquals(len(members), 1) + + if doMembershipFirst: + yield _membershipTest() + yield _membersTest() + else: + yield _membersTest() + yield _membershipTest() + + # Restore removed user + parser = XMLAccountsParser(directoryService.xmlFile) + directoryService._parsedAccounts = parser.items + + self._clearProxy(principal, proxyType) + self._clearProxy(fakePrincipal, proxyType)

1 0

[3572] CalendarServer/trunk/calendarserver/tools/fixcalendardata.py
by source_changes＠macosforge.org 06 Jan '09

06 Jan '09

Revision: 3572 http://trac.macosforge.org/projects/calendarserver/changeset/3572 Author: cdaboo(a)apple.com Date: 2009-01-06 12:23:10 -0800 (Tue, 06 Jan 2009) Log Message: ----------- Tool to scan and fix all calendar data with broken double-quote escape problems. Added Paths: ----------- CalendarServer/trunk/calendarserver/tools/fixcalendardata.py Added: CalendarServer/trunk/calendarserver/tools/fixcalendardata.py =================================================================== --- CalendarServer/trunk/calendarserver/tools/fixcalendardata.py (rev 0) +++ CalendarServer/trunk/calendarserver/tools/fixcalendardata.py 2009-01-06 20:23:10 UTC (rev 3572) @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# +## +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +from plistlib import readPlist +import time + +import datetime +import getopt +import hashlib +import os +import sys +import xattr + +PLIST_FILE = "/etc/caldavd/caldavd.plist" +SCAN_FILE = "problems.txt" + +totalProblems = 0 +totalErrors = 0 +totalScanned = 0 + +def usage(e=None): + if e: + print e + print "" + + name = os.path.basename(sys.argv[0]) + print "usage: %s [options]" % (name,) + print "" + print "Fix double-quote/escape bugs in iCalendar data." + print "" + print "options:" + print " -h --help: print this help and exit" + print " -f --config: Specify caldavd.plist configuration path" + print " -o <path>: path to file for scan results [problems.txt]" + print " --scan: Scan for problems" + print " --fix: Apply fixes" + print "" + print "One of --scan or --fix must be specified. Both may be specified" + + if e: + sys.exit(64) + else: + sys.exit(0) + +def updateEtag(path, caldata): + + x = xattr.xattr(path) + x["WebDAV:{http:%2F%2Ftwistedmatrix.com%2Fxml_namespace%2Fdav%2F}getcontentmd5"] = """<?xml version='1.0' encoding='UTF-8'?> +<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>%s</getcontentmd5> +""" % (hashlib.md5(caldata).hexdigest(),) + +def updateCtag(path): + + x = xattr.xattr(path) + x["WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"] = """<?xml version='1.0' encoding='UTF-8'?> +<getctag xmlns='http://calendarserver.org/ns/'>%s</getctag> +""" % (str(datetime.datetime.now()),) + +def parsePlist(plistPath): + + plist = readPlist(plistPath) + + try: + return plist["DocumentRoot"] + except KeyError: + raise ValueError("Unable to read DocumentRoot key from plist: %s" % (plistPath,)) + +def scanData(basePath, scanFile, doFix): + + uidsPath = os.path.join(basePath, "calendars", "__uids__") + for i in range(256): + level1Path = os.path.join(uidsPath, "%02X" % (i,)) + if os.path.exists(level1Path): + for j in range(256): + level2Path = os.path.join(level1Path, "%02X" % (j,)) + if os.path.exists(level2Path): + for item in os.listdir(level2Path): + scanCalendarHome(basePath, os.path.join(level2Path, item), scanFile, doFix) + +def scanCalendarHome(basePath, calendarHome, scanFile, doFix): + print "Scanning: %s" % (calendarHome,) + + for item in os.listdir(calendarHome): + calendarPath = os.path.join(calendarHome, item) + x = xattr.xattr(calendarPath) + if x.has_key("WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"): + scanCalendar(basePath, calendarPath, scanFile, doFix) + +def scanCalendar(basePath, calendarPath, scanFile, doFix): + + global totalProblems + global totalErrors + global totalScanned + + if not os.path.exists(calendarPath): + print "Calendar path does not exist: %s" % (calendarPath,) + totalErrors += 1 + return + + # Look at each .ics in the calendar collection + didFix = False + basePathLength = len(basePath) + for item in os.listdir(calendarPath): + if not item.endswith(".ics"): + continue + totalScanned += 1 + icsPath = os.path.join(calendarPath, item) + + try: + f = None + f = open(icsPath) + icsData = f.read() + except Exception, e: + print "Failed to read file %s due to %s" % (icsPath, str(e),) + totalErrors += 1 + continue + finally: + if f: + f.close() + + # See whether there is a \" that needs fixing. + # NB Have to handle the case of a folded line... + if icsData.find('\\"') != -1 or icsData.find('\\\r\n "') != -1: + if doFix: + if fixPath(icsPath, icsData): + didFix = True + print "Problem fixed in: <BasePath>%s" % (icsPath[basePathLength:],) + else: + print "Problem found in: <BasePath>%s" % (icsPath[basePathLength:],) + scanFile.write(icsPath + "\n") + totalProblems += 1 + + # Change CTag on calendar collection if any resource was written + if didFix: + updateCtag(calendarPath) + +def fixData(basePath, scanPath): + + global totalProblems + global totalErrors + global totalScanned + + try: + f = None + f = open(scanPath) + lines = [line[:-1] for line in f] + except Exception, e: + print "Failed to read file %s due to %s" % (scanPath, str(e),) + totalErrors += 1 + return + finally: + if f: + f.close() + + lines.sort() + calendarPaths = {} + for line in lines: + calendarPath, icsName = line.rsplit("/", 1) + calendarPaths.setdefault(calendarPath, []).append(icsName) + totalScanned += 1 + + basePathLength = len(basePath) + for calendarPath, icsNames in sorted(calendarPaths.items(), key=lambda x:x[0]): + didFix = False + for icsName in icsNames: + icsPath = os.path.join(calendarPath, icsName) + if fixPath(icsPath): + didFix = True + print "Problem fixed in: <BasePath>%s" % (icsPath[basePathLength:],) + totalProblems += 1 + + # Change CTag on calendar collection if any resource was written + if didFix: + updateCtag(calendarPath) + +def fixPath(icsPath, icsData=None): + + global totalProblems + global totalErrors + global totalScanned + + if icsData is None: + try: + f = None + f = open(icsPath) + icsData = f.read() + except Exception, e: + print "Failed to read file %s due to %s" % (icsPath, str(e),) + totalErrors += 1 + return False + finally: + if f: + f.close() + + # Fix by continuously replacing \" with " until no more replacements occur + while True: + newIcsData = icsData.replace('\\"', '"').replace('\\\r\n "', '\r\n "') + if newIcsData == icsData: + break + else: + icsData = newIcsData + + try: + f = None + f = open(icsPath, "w") + f.write(icsData) + except Exception, e: + print "Failed to write file %s due to %s" % (icsPath, str(e),) + totalErrors += 1 + return False + finally: + if f: + f.close() + + # Change ETag on written resource + updateEtag(icsPath, icsData) + + return True + +def main(): + + plistPath = PLIST_FILE + scanPath = SCAN_FILE + doScan = False + doFix = False + + # Parse command line options + opts, _ignore_args = getopt.getopt(sys.argv[1:], "f:ho:", ["config", "scan", "fix", "help",]) + for option, value in opts: + if option in ("-h", "--help"): + usage() + elif option in ("-f", "--config"): + plistPath = value + if not os.path.exists(plistPath): + usage("Path does not exist: %s" % (plistPath,)) + elif option == "-o": + scanPath = value + elif option == "--scan": + doScan = True + elif option == "--fix": + doFix = True + else: + usage("Invalid option") + + if not doScan and not doFix: + usage("Must specify one or both of --scan or --fix") + + if not doScan and doFix and not scanPath: + usage("Need to specify a file listing each path to fix") + + basePath = parsePlist(plistPath) + + start = time.time() + print "Base Path is: %s" % (basePath,) + if doScan: + if doFix: + print "Scanning data store and fixing" + scanFile = None + else: + print "Scanning data store and writing results to '%s'" % (scanPath,) + try: + scanFile = open(scanPath, "w") + except Exception, e: + print "Failed to open file for writing %s due to %s" % (scanPath, str(e),) + scanData(basePath, scanFile, doFix) + elif doFix: + print "Fixing data using results from '%s'" % (scanPath,) + fixData(basePath, scanPath) + difftime = time.time() - start + + print "" + print "---------------------" + print "Total Problems %s: %d of %d" % ("Fixed" if doFix else "Found", totalProblems, totalScanned,) + if totalErrors: + print "Total Errors: %s" % (totalErrors,) + print "Time taken (secs): %.1f" % (difftime,) + +if __name__ == '__main__': + + main() Property changes on: CalendarServer/trunk/calendarserver/tools/fixcalendardata.py ___________________________________________________________________ Added: svn:executable + *

1 0

[3571] CalendarServer/trunk
by source_changes＠macosforge.org 05 Jan '09

05 Jan '09

Revision: 3571 http://trac.macosforge.org/projects/calendarserver/changeset/3571 Author: sagen(a)apple.com Date: 2009-01-05 15:24:17 -0800 (Mon, 05 Jan 2009) Log Message: ----------- /timezones and /ischedule don't require on-disk presence Modified Paths: -------------- CalendarServer/trunk/calendarserver/tap/caldav.py CalendarServer/trunk/twistedcaldav/static.py Modified: CalendarServer/trunk/calendarserver/tap/caldav.py =================================================================== --- CalendarServer/trunk/calendarserver/tap/caldav.py 2009-01-05 18:57:18 UTC (rev 3570) +++ CalendarServer/trunk/calendarserver/tap/caldav.py 2009-01-05 23:24:17 UTC (rev 3571) @@ -64,6 +64,7 @@ from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource from twistedcaldav.directory.aggregate import AggregateDirectoryService from twistedcaldav.directory.sudo import SudoDirectoryService +from twistedcaldav.directory.util import NotFilePath from twistedcaldav.directory.wiki import WikiDirectoryService from twistedcaldav.httpfactory import HTTP503LoggingFactory, RedirectRequest from twistedcaldav.static import CalendarHomeProvisioningFile @@ -481,7 +482,7 @@ % (self.timezoneServiceResourceClass,)) timezoneService = self.timezoneServiceResourceClass( - os.path.join(config.DocumentRoot, "timezones"), + NotFilePath(isfile=True), root, ) root.putChild("timezones", timezoneService) @@ -492,7 +493,7 @@ % (self.iScheduleResourceClass,)) ischedule = self.iScheduleResourceClass( - os.path.join(config.DocumentRoot, "ischedule"), + NotFilePath(isfile=True), root, ) root.putChild("ischedule", ischedule) Modified: CalendarServer/trunk/twistedcaldav/static.py =================================================================== --- CalendarServer/trunk/twistedcaldav/static.py 2009-01-05 18:57:18 UTC (rev 3570) +++ CalendarServer/trunk/twistedcaldav/static.py 2009-01-05 23:24:17 UTC (rev 3571) @@ -895,6 +895,15 @@ (caldav_namespace, "calendar-collection-location-ok") ) + def hasDeadProperty(self, name): + return False + + def etag(self): + return None + + def checkPreconditions(self, request): + return None + ## # ACL ## @@ -902,6 +911,8 @@ def supportedPrivileges(self, request): return succeed(deliverSchedulePrivilegeSet) + + class FreeBusyURLFile (AutoProvisioningFileMixIn, FreeBusyURLResource, CalDAVFile): """ Free-busy URL resource. @@ -1007,6 +1018,18 @@ (caldav_namespace, "calendar-collection-location-ok") ) + def hasDeadProperty(self, name): + return False + + def etag(self): + return None + + def checkPreconditions(self, request): + return None + + def checkPrivileges(self, request, privileges, recurse=False, principal=None, inherited_aces=None): + return succeed(None) + ## # Utilities ##

1 0

[3570] CalendarServer/trunk/calendarserver/tools/doublequotefix.py
by source_changes＠macosforge.org 05 Jan '09

05 Jan '09

Revision: 3570 http://trac.macosforge.org/projects/calendarserver/changeset/3570 Author: cdaboo(a)apple.com Date: 2009-01-05 10:57:18 -0800 (Mon, 05 Jan 2009) Log Message: ----------- Fix invalid double-quote escaping in .ics files for REPORTs failing with a 500 error. Added Paths: ----------- CalendarServer/trunk/calendarserver/tools/doublequotefix.py Added: CalendarServer/trunk/calendarserver/tools/doublequotefix.py =================================================================== --- CalendarServer/trunk/calendarserver/tools/doublequotefix.py (rev 0) +++ CalendarServer/trunk/calendarserver/tools/doublequotefix.py 2009-01-05 18:57:18 UTC (rev 3570) @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# +## +# Copyright (c) 2008 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +import datetime +import getopt +import hashlib +import os +import sys +import xattr + +ICALSERVER_DOCROOT = "/Library/CalendarServer/Documents" +DEFAULT_URIS = "uris.txt" + +totalProblems = 0 +totalErrors = 0 +totalScanned = 0 + +def usage(e=None): + if e: + print e + print "" + + name = os.path.basename(sys.argv[0]) + print "usage: %s [options]" % (name,) + print "" + print "Fix double-quote/escape bugs in iCalendar data." + print "" + print "options:" + print " -h --help: print this help and exit" + print " -p <path>: path to calendar server document root [icalserver default]" + print " -u <path>: path to file containing uris to process [uris.txt]" + print " --fix: Apply fixes, otherwise only check for problems" + print "" + print "uris: list of uris to process" + + if e: + sys.exit(64) + else: + sys.exit(0) + +def updateEtag(path, caldata): + + x = xattr.xattr(path) + x["WebDAV:{http:%2F%2Ftwistedmatrix.com%2Fxml_namespace%2Fdav%2F}getcontentmd5"] = """<?xml version='1.0' encoding='UTF-8'?> +<getcontentmd5 xmlns='http://twistedmatrix.com/xml_namespace/dav/'>%s</getcontentmd5> +""" % (hashlib.md5(caldata).hexdigest(),) + +def updateCtag(path): + + x = xattr.xattr(path) + x["WebDAV:{http:%2F%2Fcalendarserver.org%2Fns%2F}getctag"] = """<?xml version='1.0' encoding='UTF-8'?> +<getctag xmlns='http://calendarserver.org/ns/'>%s</getctag> +""" % (str(datetime.datetime.now()),) + +def scanURI(uri, basePath, doFix): + + global totalProblems + global totalErrors + global totalScanned + + # Verify we have a valid path + pathBits = uri.strip("/").rstrip("/").split("/") + if len(pathBits) != 4 or pathBits[0] != "calendars" or pathBits[1] != "__uids__": + print "Invalid uri (ignoring): %s" % (uri,) + totalErrors += 1 + return + + # Absolute hashed directory path to calendar collection + calendarPath = os.path.join( + basePath, + pathBits[0], + pathBits[1], + pathBits[2][0:2], + pathBits[2][2:4], + pathBits[2], + pathBits[3], + ) + + if not os.path.exists(calendarPath): + print "Calendar path does not exist: %s" % (calendarPath,) + totalErrors += 1 + return + + # Look at each .ics in the calendar collection + didFix = False + basePathLength = len(basePath) + for item in os.listdir(calendarPath): + if not item.endswith(".ics"): + continue + totalScanned += 1 + icsPath = os.path.join(calendarPath, item) + + try: + f = open(icsPath) + icsData = f.read() + except Exception, e: + print "Failed to read file %s due to %s" % (icsPath, str(e),) + totalErrors += 1 + continue + finally: + f.close() + + # See whether there is a \" that needs fixing. + # NB Have to handle the case of a folded line... + if icsData.find('\\"') != -1 or icsData.find('\\\r\n "') != -1: + if doFix: + # Fix by continuously replacing \" with " until no more replacements occur + while True: + newIcsData = icsData.replace('\\"', '"').replace('\\\r\n "', '\r\n "') + if newIcsData == icsData: + break + else: + icsData = newIcsData + + try: + f = open(icsPath, "w") + f.write(icsData) + except Exception, e: + print "Failed to write file %s due to %s" % (icsPath, str(e),) + totalErrors += 1 + continue + finally: + f.close() + + # Change ETag on written resource + updateEtag(icsPath, icsData) + didFix = True + print "Problem fixed in: <BasePath>%s" % (icsPath[basePathLength:],) + else: + print "Problem found in: <BasePath>%s" % (icsPath[basePathLength:],) + totalProblems += 1 + + # Change CTag on calendar collection if any resource was written + if didFix: + updateCtag(calendarPath) + +def main(): + + basePath = ICALSERVER_DOCROOT + urisFile = DEFAULT_URIS + doFix = False + + # Parse command line options + opts, _ignore_args = getopt.getopt(sys.argv[1:], "hp:u:", ["fix", "help",]) + for option, value in opts: + if option in ("-h", "--help"): + usage() + elif option == "-p": + basePath = value + if not os.path.exists(basePath): + usage("Path does not exist: %s" % (basePath,)) + elif not os.path.isdir(basePath): + usage("Path is not a directory: %s" % (basePath,)) + elif option == "-u": + urisFile = value + elif option == "--fix": + doFix = True + else: + usage("Invalid option") + + if not urisFile: + usage("Need to specify a file listing each URI to process") + + # Get all the uris to process + f = open(urisFile) + uris = set() + for line in f: + pos = line.find("/calendars/") + if pos == -1: + print "Ignored log line: %s" % (line,) + continue + uris.add(line[pos:].split()[0]) + uris = list(uris) + uris.sort() + f.close() + + print "Base Path is: %s" % (basePath,) + print "Number of unique URIs to fix: %d" % (len(uris),) + print "" + for uri in uris: + scanURI(uri, basePath, doFix) + + print "" + print "---------------------" + print "Total Problems %s: %d of %d" % ("Fixed" if doFix else "Found", totalProblems, totalScanned,) + if totalErrors: + print "Total Errors: %s" % (totalErrors,) + +if __name__ == '__main__': + + main() Property changes on: CalendarServer/trunk/calendarserver/tools/doublequotefix.py ___________________________________________________________________ Added: svn:executable + *

1 0