Revision: 9297 http://trac.macosforge.org/projects/calendarserver/changeset/9297 Author: glyph@apple.com Date: 2012-05-24 13:40:14 -0700 (Thu, 24 May 2012) Log Message: ----------- Merge uuid-normalize branch. This adds an upgrader to the data store to make sure that all UIDs in the data store which are also directory GUIDs are stored in a canonical UUID form. This corresponds to the change made in r8549 to normalize values as they come out of the directory. Revision Links: -------------- http://trac.macosforge.org/projects/calendarserver/changeset/8549 Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/database.py CalendarServer/trunk/twistedcaldav/directory/augment.py CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py CalendarServer/trunk/twistedcaldav/mail.py CalendarServer/trunk/txdav/base/datastore/util.py CalendarServer/trunk/txdav/caldav/datastore/sql.py CalendarServer/trunk/txdav/caldav/datastore/util.py CalendarServer/trunk/txdav/common/datastore/sql.py CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql CalendarServer/trunk/txdav/common/datastore/test/test_sql.py Added Paths: ----------- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py Property Changed: ---------------- CalendarServer/trunk/ Property changes on: CalendarServer/trunk ___________________________________________________________________ Modified: svn:mergeinfo - /CalendarServer/branches/config-separation:4379-4443 /CalendarServer/branches/egg-info-351:4589-4625 /CalendarServer/branches/generic-sqlstore:6167-6191 /CalendarServer/branches/new-store:5594-5934 /CalendarServer/branches/new-store-no-caldavfile:5911-5935 /CalendarServer/branches/new-store-no-caldavfile-2:5936-5981 /CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198 /CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702 /CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346 /CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644 /CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141 /CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601 /CalendarServer/branches/users/cdaboo/partition-4464:4465-4957 /CalendarServer/branches/users/cdaboo/pods:7297-7377 /CalendarServer/branches/users/cdaboo/pycalendar:7085-7206 /CalendarServer/branches/users/cdaboo/pycard:7227-7237 /CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287 /CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105 /CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440 /CalendarServer/branches/users/cdaboo/timezones:7443-7699 /CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743 /CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805 /CalendarServer/branches/users/glyph/conn-limit:6574-6577 /CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080 /CalendarServer/branches/users/glyph/dalify:6932-7023 /CalendarServer/branches/users/glyph/db-reconnect:6824-6876 /CalendarServer/branches/users/glyph/deploybuild:7563-7572 /CalendarServer/branches/users/glyph/disable-quota:7718-7727 /CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614 /CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984 /CalendarServer/branches/users/glyph/ipv6-client:9054-9105 /CalendarServer/branches/users/glyph/linux-tests:6893-6900 /CalendarServer/branches/users/glyph/migrate-merge:8690-8713 /CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374 /CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368 /CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445 /CalendarServer/branches/users/glyph/multiget-delete:8321-8330 /CalendarServer/branches/users/glyph/new-export:7444-7485 /CalendarServer/branches/users/glyph/oracle:7106-7155 /CalendarServer/branches/users/glyph/oracle-nulls:7340-7351 /CalendarServer/branches/users/glyph/other-html:8062-8091 /CalendarServer/branches/users/glyph/parallel-sim:8240-8251 /CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400 /CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583 /CalendarServer/branches/users/glyph/quota:7604-7637 /CalendarServer/branches/users/glyph/sendfdport:5388-5424 /CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443 /CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174 /CalendarServer/branches/users/glyph/sharedpool:6490-6550 /CalendarServer/branches/users/glyph/sharing-api:9192-9205 /CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535 /CalendarServer/branches/users/glyph/sql-store:5929-6073 /CalendarServer/branches/users/glyph/subtransactions:7248-7258 /CalendarServer/branches/users/glyph/table-alias:8651-8664 /CalendarServer/branches/users/glyph/uidexport:7673-7676 /CalendarServer/branches/users/glyph/use-system-twisted:5084-5149 /CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769 /CalendarServer/branches/users/sagen/applepush:8126-8184 /CalendarServer/branches/users/sagen/inboxitems:7380-7381 /CalendarServer/branches/users/sagen/locations-resources:5032-5051 /CalendarServer/branches/users/sagen/locations-resources-2:5052-5061 /CalendarServer/branches/users/sagen/purge_old_events:6735-6746 /CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067 /CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075 /CalendarServer/branches/users/sagen/resources-2:5084-5093 /CalendarServer/branches/users/wsanchez/transations:5515-5593 + /CalendarServer/branches/config-separation:4379-4443 /CalendarServer/branches/egg-info-351:4589-4625 /CalendarServer/branches/generic-sqlstore:6167-6191 /CalendarServer/branches/new-store:5594-5934 /CalendarServer/branches/new-store-no-caldavfile:5911-5935 /CalendarServer/branches/new-store-no-caldavfile-2:5936-5981 /CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198 /CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702 /CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346 /CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644 /CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141 /CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601 /CalendarServer/branches/users/cdaboo/partition-4464:4465-4957 /CalendarServer/branches/users/cdaboo/pods:7297-7377 /CalendarServer/branches/users/cdaboo/pycalendar:7085-7206 /CalendarServer/branches/users/cdaboo/pycard:7227-7237 /CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287 /CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105 /CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440 /CalendarServer/branches/users/cdaboo/timezones:7443-7699 /CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743 /CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805 /CalendarServer/branches/users/glyph/conn-limit:6574-6577 /CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080 /CalendarServer/branches/users/glyph/dalify:6932-7023 /CalendarServer/branches/users/glyph/db-reconnect:6824-6876 /CalendarServer/branches/users/glyph/deploybuild:7563-7572 /CalendarServer/branches/users/glyph/disable-quota:7718-7727 /CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614 /CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984 /CalendarServer/branches/users/glyph/ipv6-client:9054-9105 /CalendarServer/branches/users/glyph/linux-tests:6893-6900 /CalendarServer/branches/users/glyph/migrate-merge:8690-8713 /CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374 /CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368 /CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445 /CalendarServer/branches/users/glyph/multiget-delete:8321-8330 /CalendarServer/branches/users/glyph/new-export:7444-7485 /CalendarServer/branches/users/glyph/oracle:7106-7155 /CalendarServer/branches/users/glyph/oracle-nulls:7340-7351 /CalendarServer/branches/users/glyph/other-html:8062-8091 /CalendarServer/branches/users/glyph/parallel-sim:8240-8251 /CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400 /CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583 /CalendarServer/branches/users/glyph/quota:7604-7637 /CalendarServer/branches/users/glyph/sendfdport:5388-5424 /CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443 /CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174 /CalendarServer/branches/users/glyph/sharedpool:6490-6550 /CalendarServer/branches/users/glyph/sharing-api:9192-9205 /CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535 /CalendarServer/branches/users/glyph/sql-store:5929-6073 /CalendarServer/branches/users/glyph/subtransactions:7248-7258 /CalendarServer/branches/users/glyph/table-alias:8651-8664 /CalendarServer/branches/users/glyph/uidexport:7673-7676 /CalendarServer/branches/users/glyph/use-system-twisted:5084-5149 /CalendarServer/branches/users/glyph/uuid-normalize:9268-9296 /CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769 /CalendarServer/branches/users/sagen/applepush:8126-8184 /CalendarServer/branches/users/sagen/inboxitems:7380-7381 /CalendarServer/branches/users/sagen/locations-resources:5032-5051 /CalendarServer/branches/users/sagen/locations-resources-2:5052-5061 /CalendarServer/branches/users/sagen/purge_old_events:6735-6746 /CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067 /CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075 /CalendarServer/branches/users/sagen/resources-2:5084-5093 /CalendarServer/branches/users/wsanchez/transations:5515-5593 Modified: CalendarServer/trunk/twistedcaldav/database.py =================================================================== --- CalendarServer/trunk/twistedcaldav/database.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/twistedcaldav/database.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -349,6 +349,7 @@ # cannot be thrown away. raise NotImplementedError("Persistent databases MUST support an upgrade method.") + @inlineCallbacks def _db_upgrade_schema(self): """ @@ -356,6 +357,7 @@ """ yield self._db_execute("insert or replace into CALDAV (KEY, VALUE) values ('SCHEMA_VERSION', :1)", (self._db_version(),)) + @inlineCallbacks def _db_remove(self): """ Modified: CalendarServer/trunk/twistedcaldav/directory/augment.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/augment.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/twistedcaldav/directory/augment.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -31,6 +31,7 @@ from twistedcaldav.directory.xmlaugmentsparser import XMLAugmentsParser from twistedcaldav.xmlutil import newElementTreeWithRoot, addSubElement,\ writeXML, readXML +from twistedcaldav.directory.util import normalizeUUID log = Logger() @@ -88,8 +89,35 @@ def __init__(self): self.cachedRecords = {} - + + @inlineCallbacks + def normalizeUUIDs(self): + """ + Normalize (uppercase) all augment UIDs which are parseable as UUIDs. + + @return: a L{Deferred} that fires when all records have been + normalized. + """ + remove = [] + add = [] + for uid in (yield self.getAllUIDs()): + nuid = normalizeUUID(uid) + if uid != nuid: + old = yield self._lookupAugmentRecord(uid) + new = copy.deepcopy(old) + new.uid = uid.upper() + remove.append(old) + add.append(new) + try: + yield self.removeAugmentRecords(remove) + yield self.addAugmentRecords(add) + except IOError: + # It's OK if we can't re-write the file. + pass + + + @inlineCallbacks def getAugmentRecord(self, uid, recordType): """ Get an AugmentRecord for the specified UID or the default. @@ -242,18 +270,18 @@ raise self.lastCached = time.time() + self.normalizeUUIDs() - @inlineCallbacks def getAllUIDs(self): """ Get all AugmentRecord UIDs. @return: L{Deferred} """ - return succeed(self.db.keys()) + def _lookupAugmentRecord(self, uid): """ Get an AugmentRecord for the specified UID. Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -47,6 +47,7 @@ from twistedcaldav.directory.principal import formatLinks from twistedcaldav.directory.principal import formatPrincipals +from twistedcaldav.directory.util import normalizeUUID from twistedcaldav.config import config, fullServerPath from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\ ADBAPIPostgreSQLMixin @@ -415,11 +416,11 @@ """ - schema_version = "4" + schema_version = "5" schema_type = "CALENDARUSERPROXY" - + class ProxyDBMemcacher(Memcacher): - + def __init__(self, namespace): super(ProxyDB.ProxyDBMemcacher, self).__init__(namespace, key_normalization=config.Memcached.ProxyDBKeyNormalization) @@ -774,6 +775,7 @@ ifnotexists=True, ) + @inlineCallbacks def _db_upgrade_data_tables(self, old_version): """ @@ -797,6 +799,18 @@ ifnotexists=True, ) + if int(old_version) < 5: + for (groupname, member) in ( + (yield self._db_all_values_for_sql("select GROUPNAME, MEMBER from GROUPS")) + ): + grouplist = groupname.split("#") + grouplist[0] = normalizeUUID(grouplist[0]) + yield self._db_execute(""" + update GROUPS set GROUPNAME = :1, MEMBER = :2 + where GROUPNAME = :1 and MEMBER = :2 + """, ["#".join(grouplist), normalizeUUID(member)]) + + def _db_empty_data_tables(self): """ Empty the underlying database tables. Modified: CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py =================================================================== --- CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/twistedcaldav/directory/xmlaccountsparser.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -176,15 +176,15 @@ def expand(text, ctr): """ Returns a string where ~<number> is replaced by the first <number> - characters from the md5 hexdigest of str(ctr), e.g.: + characters from the md5 hexdigest of str(ctr), e.g.:: expand("~9 foo", 1) - returns: + returns:: "c4ca4238a foo" - ...since "c4ca4238a" is the first 9 characters of: + ...since "c4ca4238a" is the first 9 characters of:: hashlib.md5(str(1)).hexdigest() @@ -258,7 +258,9 @@ self.shortNames.append(child.firstChild.data.encode("utf-8")) elif child_name == ELEMENT_GUID: if child.firstChild is not None: - self.guid = child.firstChild.data.encode("utf-8") + self.guid = normalizeUUID( + child.firstChild.data.encode("utf-8") + ) if len(self.guid) < 4: self.guid += "?" * (4 - len(self.guid)) elif child_name == ELEMENT_PASSWORD: Modified: CalendarServer/trunk/twistedcaldav/mail.py =================================================================== --- CalendarServer/trunk/twistedcaldav/mail.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/twistedcaldav/mail.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -744,7 +744,12 @@ ) self._db_commit() + def lowercase(self): + """ + Lowercase mailto: addresses (and uppercase urn:uuid: addresses!) so + they can be located via normalized names. + """ rows = self._db_execute( """ select ORGANIZER, ATTENDEE from TOKENS @@ -759,6 +764,14 @@ update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 """, organizer.lower(), organizer ) + else: + from txdav.base.datastore.util import normalizeUUIDOrNot + self._db_execute( + """ + update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 + """, normalizeUUIDOrNot(organizer), organizer + ) + # ATTENDEEs are always mailto: so unconditionally lower(). self._db_execute( """ update TOKENS set ATTENDEE = :1 WHERE ATTENDEE = :2 @@ -766,6 +779,7 @@ ) self._db_commit() + def _db_version(self): """ @return: the schema version assigned to this index. Modified: CalendarServer/trunk/txdav/base/datastore/util.py =================================================================== --- CalendarServer/trunk/txdav/base/datastore/util.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/txdav/base/datastore/util.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -19,6 +19,10 @@ Common utility functions for a datastores. """ +from uuid import UUID + +from twisted.python import log + from twistedcaldav.memcacher import Memcacher _unset = object() @@ -90,3 +94,51 @@ def keyForHomeChildMetaData(self, resourceID): return "homeChildMetaData:%s" % (resourceID) + + +def normalizeUUIDOrNot(somestr): + """ + Take a string which may be: + + - the hex format of a UUID + + - a urn:uuid: URI containing a UUID + + - some other random thing + + and return, respectively: + + - the hex format of a UUID converted to upper case + + - a urn:uuid: URI with an upper-cased UUID (but not an upper-cased + scheme and namespace) + + - some other random thing, unmodified + + @type somestr: L{str} + + @return: L{str} + """ + uuu = "urn:uuid:" + isURI = somestr.startswith(uuu) + if isURI: + normstr = somestr[len(uuu):] + else: + normstr = somestr + try: + uu = UUID(normstr) + except ValueError: + if isURI: + log.msg(format="normalizing urn:uuid: without UUID: %(uid)r", + uid=somestr) + # not a UUID, whatever + return somestr + else: + normalForm = str(uu).upper() + if isURI: + return uuu + normalForm + else: + return normalForm + + + Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py =================================================================== --- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -508,7 +508,7 @@ ).on(self._txn) self._supportedComponents = supported_components - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher is not None: cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID) yield queryCacher.invalidateAfterCommit(self._txn, cacheKey) Modified: CalendarServer/trunk/txdav/caldav/datastore/util.py =================================================================== --- CalendarServer/trunk/txdav/caldav/datastore/util.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/txdav/caldav/datastore/util.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -14,34 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. ## -from twistedcaldav.config import config """ Utility logic common to multiple backend implementations. """ +import os + +from zope.interface.declarations import implements + +from txdav.caldav.icalendarstore import IAttachmentStorageTransport + +from twisted.python.failure import Failure +from twisted.internet.defer import inlineCallbacks, Deferred, returnValue +from twisted.internet.protocol import Protocol + from twext.python.log import Logger + +from twext.web2 import http_headers + from twext.python.vcomponent import InvalidICalendarDataError from twext.python.vcomponent import VComponent -from twext.web2 import http_headers -from twisted.internet.defer import inlineCallbacks, Deferred, returnValue -from twisted.internet.protocol import Protocol - from twistedcaldav.datafilters.peruserdata import PerUserDataFilter from twistedcaldav.datafilters.privateevents import PrivateEventFilter -from txdav.caldav.icalendarstore import IAttachmentStorageTransport - from txdav.common.icommondatastore import ( InvalidObjectResourceError, NoSuchObjectResourceError, InternalDataStoreError, HomeChildNameAlreadyExistsError ) +from txdav.base.datastore.util import normalizeUUIDOrNot -from zope.interface.declarations import implements - -import os - log = Logger() validationBypass = False @@ -88,6 +91,7 @@ raise InvalidObjectResourceError(e) + @inlineCallbacks def dropboxIDFromCalendarObject(calendarObject): """ @@ -135,6 +139,7 @@ returnValue(uid + ".dropbox") + @inlineCallbacks def _migrateCalendar(inCalendar, outCalendar, getComponent, merge=False): """ @@ -170,7 +175,6 @@ bad_count += 1 continue - if ctype not in ("VEVENT", "VTODO"): log.error("Migration skipping unsupported (%s) calendar object %r" % (ctype, calendarObject)) @@ -241,6 +245,7 @@ returnValue((ok_count, bad_count,)) + # MIME helpers - mostly copied from twext.web2.static def loadMimeTypes(mimetype_locations=['/etc/mime.types']): @@ -280,12 +285,15 @@ return contentTypes + + def getType(filename, types, defaultType="application/octet-stream"): _ignore_p, ext = os.path.splitext(filename) ext = ext.lower() return types.get(ext, defaultType) + class _AttachmentMigrationProto(Protocol, object): def __init__(self, storeTransport): self.storeTransport = storeTransport @@ -329,6 +337,7 @@ @return: a L{Deferred} that fires with C{None} when the migration is complete. """ + from twistedcaldav.config import config if not merge: yield outHome.removeCalendarWithName("calendar") if config.RestrictCalendarsToOneComponentType: @@ -434,6 +443,7 @@ return '<Storing Attachment: %r%s>' % (self.attachment.name(), host) + class StorageTransportBase(object): """ Base logic shared between file- and sql-based L{IAttachmentStorageTransport} @@ -467,3 +477,57 @@ def writeSequence(self, seq): return self.write(''.join(seq)) + + + +def fixOneCalendarObject(component): + """ + Correct the properties which may contain a user's directory UUID within a + single calendar component, by normalizing the directory UUID. + + @param component: any iCalendar component. + @type component: L{twistedcaldav.ical.Component} + + @return: a 2-tuple of the number of fixes performed and the new + L{Component} + """ + fixes = 0 + for calprop in component.properties(): + if calprop.name() in ( + "ATTENDEE", "ORGANIZER", PerUserDataFilter.PERUSER_UID + ): + preval = calprop.value() + postval = normalizeUUIDOrNot(preval) + if preval != postval: + fixes += 1 + calprop.setValue(postval) + for subc in component.subcomponents(): + count, fixsubc = fixOneCalendarObject(subc) + fixes += count + return fixes, component + + + +@inlineCallbacks +def fixOneCalendarHome(home): + """ + Correct the case of UIDs on one L{ICalendarHome}. + + @return: a L{Deferred} that fires with the number of fixes made when the + fixes are complete. + """ + fixedThisHome = 0 + for calendar in (yield home.calendars()): + for calObj in (yield calendar.calendarObjects()): + try: + comp = (yield calObj.component()) + fixCount, comp = fixOneCalendarObject(comp) + fixedThisHome += fixCount + if fixCount: + yield calObj.setComponent(comp) + except: + log.err(Failure(), + 'Error while processing calendar/object %r %r' % ( + calendar.name(), calObj.name() + )) + returnValue(fixedThisHome) Modified: CalendarServer/trunk/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/trunk/txdav/common/datastore/sql.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/txdav/common/datastore/sql.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -25,11 +25,12 @@ "CommonHome", ] -from uuid import uuid4 +from uuid import uuid4, UUID from zope.interface import implements, directlyProvides from twext.python.log import Logger, LoggingMixIn +from twisted.python.log import msg as log_msg, err as log_err from txdav.xml.rfc2518 import ResourceType from txdav.xml.parser import WebDAVDocument from twext.web2.http_headers import MimeType @@ -66,15 +67,11 @@ from twext.python.clsprop import classproperty from twext.enterprise.ienterprise import AlreadyFinishedError from twext.enterprise.dal.parseschema import significant -from twext.enterprise.dal.syntax import Delete, utcNowSQL, Union -from twext.enterprise.dal.syntax import Insert -from twext.enterprise.dal.syntax import Len -from twext.enterprise.dal.syntax import Max -from twext.enterprise.dal.syntax import Parameter -from twext.enterprise.dal.syntax import SavepointAction -from twext.enterprise.dal.syntax import Select -from twext.enterprise.dal.syntax import Update +from twext.enterprise.dal.syntax import \ + Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction, \ + Select, Update, ColumnSyntax, TableSyntax, Upper + from txdav.base.propertystore.base import PropertyName from txdav.base.propertystore.none import PropertyStore as NonePropertyStore from txdav.base.propertystore.sql import PropertyStore @@ -85,6 +82,8 @@ pyCalendarTodatetime from txdav.xml.rfc2518 import DisplayName +from txdav.base.datastore.util import normalizeUUIDOrNot + from cStringIO import StringIO from sqlparse import parse import collections @@ -97,6 +96,7 @@ ECALENDARTYPE = 0 EADDRESSBOOKTYPE = 1 +ENOTIFICATIONTYPE = 2 # Labels used to identify the class of resource being modified, so that # notification systems can target the correct application @@ -181,7 +181,7 @@ return [] - def newTransaction(self, label="unlabeled"): + def newTransaction(self, label="unlabeled", disableCache=False): """ @see: L{IDataStore.newTransaction} """ @@ -193,12 +193,14 @@ self.notifierFactory if self._enableNotifications else None, label, self._migrating, + disableCache ) - if self.logTransactionWaits or self.timeoutTransactions: - CommonStoreTransactionMonitor(txn, self.logTransactionWaits, self.timeoutTransactions) + CommonStoreTransactionMonitor(txn, self.logTransactionWaits, + self.timeoutTransactions) return txn + def setMigrating(self, state): """ Set the "migrating" state @@ -306,7 +308,7 @@ def __init__(self, store, sqlTxn, enableCalendars, enableAddressBooks, - notifierFactory, label, migrating=False): + notifierFactory, label, migrating=False, disableCache=False): self._store = store self._calendarHomes = {} self._addressbookHomes = {} @@ -318,6 +320,11 @@ self._label = label self._migrating = migrating self._primaryHomeType = None + self._disableCache = disableCache + if disableCache: + self._queryCacher = None + else: + self._queryCacher = store.queryCacher CommonStoreTransaction.id += 1 self._txid = CommonStoreTransaction.id @@ -840,6 +847,15 @@ returnValue(count) +class _EmptyCacher(object): + def set(self, key, value): + return succeed(True) + def get(self, key, withIdentifier=False): + return succeed(None) + def delete(self, key): + return succeed(True) + + class CommonHome(LoggingMixIn): # All these need to be initialized by derived classes for each store type @@ -871,6 +887,8 @@ self._created = None self._modified = None self._syncTokenRevision = None + if transaction._disableCache: + self._cacher = _EmptyCacher() # Needed for REVISION/BIND table join self._revisionBindJoinTable = {} @@ -931,7 +949,7 @@ if result: self._resourceID = result[0][0] - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher: # Get cached copy cacheKey = queryCacher.keyForHomeMetaData(self._resourceID) @@ -1513,7 +1531,7 @@ try: self._modified = (yield self._txn.subtransaction(_bumpModified, retries=0, failureOK=True))[0][0] - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher is not None: cacheKey = queryCacher.keyForHomeMetaData(self._resourceID) yield queryCacher.invalidateAfterCommit(self._txn, cacheKey) @@ -2279,7 +2297,7 @@ exists. """ data = None - queryCacher = home._txn.store().queryCacher + queryCacher = home._txn._queryCacher # Only caching non-shared objects so that we don't need to invalidate # in sql_legacy if owned and queryCacher: @@ -2441,7 +2459,7 @@ resource ID. We read in and cache all the extra metadata from the DB to avoid having to do DB queries for those individually later. """ - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher: # Retrieve from cache cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID) @@ -2515,7 +2533,7 @@ """ oldName = self._name - queryCacher = self._home._txn.store().queryCacher + queryCacher = self._home._txn._queryCacher if queryCacher: cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, oldName) yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey) @@ -2548,7 +2566,7 @@ @inlineCallbacks def remove(self): - queryCacher = self._home._txn.store().queryCacher + queryCacher = self._home._txn._queryCacher if queryCacher: cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name) yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey) @@ -2975,7 +2993,7 @@ try: self._modified = (yield self._txn.subtransaction(_bumpModified, retries=0, failureOK=True))[0][0] - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher is not None: cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID) yield queryCacher.invalidateAfterCommit(self._txn, cacheKey) @@ -3942,3 +3960,292 @@ +def determineNewest(uid, homeType): + """ + Construct a query to determine the modification time of the newest object + in a given home. + + @param uid: the UID of the home to scan. + @type uid: C{str} + + @param homeType: The type of home to scan; C{ECALENDARTYPE}, + C{ENOTIFICATIONTYPE}, or C{EADDRESSBOOKTYPE}. + @type homeType: C{int} + + @return: A select query that will return a single row containing a single + column which is the maximum value. + @rtype: L{Select} + """ + if homeType == ENOTIFICATIONTYPE: + return Select( + [Max(schema.NOTIFICATION.MODIFIED)], + From=schema.NOTIFICATION_HOME.join( + schema.NOTIFICATION, + on=schema.NOTIFICATION_HOME.RESOURCE_ID == + schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID), + Where=schema.NOTIFICATION_HOME.OWNER_UID == uid + ) + homeTypeName = {ECALENDARTYPE: "CALENDAR", + EADDRESSBOOKTYPE: "ADDRESSBOOK"}[homeType] + home = getattr(schema, homeTypeName + "_HOME") + bind = getattr(schema, homeTypeName + "_BIND") + child = getattr(schema, homeTypeName) + obj = getattr(schema, homeTypeName + "_OBJECT") + return Select( + [Max(obj.MODIFIED)], + From=home.join(bind, on=bind.HOME_RESOURCE_ID == home.RESOURCE_ID) + .join(child, on=child.RESOURCE_ID == bind.RESOURCE_ID) + .join(obj, on=obj.PARENT_RESOURCE_ID == child.RESOURCE_ID), + Where=(bind.BIND_MODE == 0).And(home.OWNER_UID == uid) + ) + + + +@inlineCallbacks +def mergeHomes(sqlTxn, one, other, homeType): + """ + Merge two homes together. This determines which of C{one} or C{two} is + newer - that is, has been modified more recently - and pulls all the data + from the older into the newer home. Then, it changes the UID of the old + home to its UID, normalized and prefixed with "old.", and then re-names the + new home to its name, normalized. + + Because the UIDs of both homes have changed, B{both one and two will be + invalid to all other callers from the start of the invocation of this + function}. + + @param sqlTxn: the transaction to use + @type sqlTxn: A L{CommonTransaction} + + @param one: A calendar home. + @type one: L{ICalendarHome} + + @param two: Another, different calendar home. + @type two: L{ICalendarHome} + + @param homeType: The type of home to scan; L{ECALENDARTYPE} or + L{EADDRESSBOOKTYPE}. + @type homeType: C{int} + + @return: a L{Deferred} which fires with with the newer of C{one} or C{two}, + into which the data from the other home has been merged, when the merge + is complete. + """ + from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome + from txdav.carddav.datastore.util import migrateHome as migrateABHome + migrateHome = {EADDRESSBOOKTYPE: migrateABHome, + ECALENDARTYPE: migrateCalendarHome, + ENOTIFICATIONTYPE: _dontBotherWithNotifications}[homeType] + homeTable = {EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME, + ECALENDARTYPE: schema.CALENDAR_HOME, + ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME}[homeType] + both = [] + both.append([one, + (yield determineNewest(one.uid(), homeType).on(sqlTxn))]) + both.append([other, + (yield determineNewest(other.uid(), homeType).on(sqlTxn))]) + both.sort(key=lambda x: x[1]) + # Note: determineNewest may return None sometimes. + older = both[0][0] + newer = both[1][0] + yield migrateHome(older, newer, merge=True) + # Rename the old one to 'old.<correct-guid>' + newNormalized = normalizeUUIDOrNot(newer.uid()) + oldNormalized = normalizeUUIDOrNot(older.uid()) + yield Update({homeTable.OWNER_UID: "old." + oldNormalized}, + Where=homeTable.OWNER_UID == older.uid()).on(sqlTxn) + # Rename the new one to '<correct-guid>' + if newer.uid() != newNormalized: + yield Update( + {homeTable.OWNER_UID: newNormalized}, + Where=homeTable.OWNER_UID == newer.uid() + ).on(sqlTxn) + yield returnValue(newer) + + + +def _dontBotherWithNotifications(older, newer, merge): + """ + Notifications are more transient and can be easily worked around; don't + bother to migrate all of them when there is a UUID case mismatch. + """ + + + +@inlineCallbacks +def _normalizeHomeUUIDsIn(t, homeType): + """ + Normalize the UUIDs in the given L{txdav.common.datastore.CommonStore}. + + This changes the case of the UUIDs in the calendar home. + + @param t: the transaction to normalize all the UUIDs in. + @type t: L{CommonStoreTransaction} + + @param homeType: The type of home to scan, L{ECALENDARTYPE}, + L{EADDRESSBOOKTYPE}, or L{ENOTIFICATIONTYPE}. + @type homeType: C{int} + + @return: a L{Deferred} which fires with C{None} when the UUID normalization + is complete. + """ + from txdav.caldav.datastore.util import fixOneCalendarHome + homeTable = {EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME, + ECALENDARTYPE: schema.CALENDAR_HOME, + ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME}[homeType] + homeTypeName = homeTable.model.name.split("_")[0] + + allUIDs = yield Select([homeTable.OWNER_UID], + From=homeTable, + OrderBy=homeTable.OWNER_UID).on(t) + total = len(allUIDs) + allElapsed = [] + for n, [UID] in enumerate(allUIDs): + start = time.time() + if allElapsed: + estimate = "%0.3d" % ((sum(allElapsed) / len(allElapsed)) * + total - n) + else: + estimate = "unknown" + log_msg( + format="Scanning UID %(uid)s [%(homeType)s] " + "(%(pct)0.2d%%, %(estimate)s seconds remaining)...", + uid=UID, pct=(n / float(total)) * 100, estimate=estimate, + homeType=homeTypeName + ) + other = None + if homeType == ENOTIFICATIONTYPE: + this = yield t.notificationsWithUID(UID) + else: + this = yield t.homeWithUID(homeType, UID) + if homeType == ECALENDARTYPE: + fixedThisHome = yield fixOneCalendarHome(this) + else: + fixedThisHome = 0 + fixedOtherHome = 0 + if this is None: + log_msg(format="%(uid)r appears to be missing, already processed", + uid=UID) + try: + uuidobj = UUID(UID) + except ValueError: + pass + else: + newname = str(uuidobj).upper() + if UID != newname: + log_msg(format="Detected case variance: %(uid)s %(newuid)s" + "[%(homeType)s]", + uid=UID, newuid=newname, homeType=homeTypeName) + other = yield t.homeWithUID(homeType, newname) + if homeType == ECALENDARTYPE: + fixedOtherHome = yield fixOneCalendarHome(other) + if other is not None: + this = yield mergeHomes(t, this, other, homeType) + # NOTE: WE MUST NOT TOUCH EITHER HOME OBJECT AFTER THIS + # POINT. THE UIDS HAVE CHANGED AND ALL OPERATIONS WILL + # FAIL. + + end = time.time() + elapsed = end - start + allElapsed.append(elapsed) + log_msg(format="Scanned UID %(uid)s; %(elapsed)s seconds elapsed," + " %(fixes)s properties fixed (%(duplicate)s fixes in " + "duplicate).", uid=UID, elapsed=elapsed, fixes=fixedThisHome, + duplicate=fixedOtherHome) + returnValue(None) + + + +@inlineCallbacks +def _normalizeColumnUUIDs(txn, column): + """ + Upper-case the UUIDs in the given SQL DAL column. + + @param txn: The transaction. + @type txn: L{CommonStoreTransaction} + + @param column: the column, which may contain UIDs, to normalize. + @type column: L{ColumnSyntax} + + @return: A L{Deferred} that will fire when the UUID normalization of the + given column has completed. + """ + tableModel = column.model.table + # Get a primary key made of column syntax objects for querying and + # comparison later. + pkey = [ColumnSyntax(columnModel) + for columnModel in tableModel.primaryKey] + for row in (yield Select([column] + pkey, + From=TableSyntax(tableModel)).on(txn)): + before = row[0] + pkeyparts = row[1:] + after = normalizeUUIDOrNot(before) + if after != before: + where = _AndNothing + # Build a where clause out of the primary key and the parts of the + # primary key that were found. + for pkeycol, pkeypart in zip(pkeyparts, pkey): + where = where.And(pkeycol == pkeypart) + yield Update({column: after}, Where=where).on(txn) + + + +class _AndNothing(object): + """ + Simple placeholder for iteratively generating a 'Where' clause; the 'And' + just returns its argument, so it can be used at the start of the loop. + """ + @staticmethod + def And(self): + """ + Return the argument. + """ + return self + + + +@inlineCallbacks +def fixUUIDNormalization(store): + """ + Fix all UUIDs in the given SQL store to be in a canonical form; + 00000000-0000-0000-0000-000000000000 format and upper-case. + """ + t = store.newTransaction(disableCache=True) + + # First, let's see if there are any calendar or addressbook homes that have + # a lower-case OWNER_UID. If there are none, then we can early-out and + # avoid the tedious and potentially expensive inspection of oodles of + # calendar data. + for x in [schema.CALENDAR_HOME, schema.ADDRESSBOOK_HOME]: + slct = Select([x.OWNER_UID], From=x, + Where=x.OWNER_UID != Upper(x.OWNER_UID)) + rows = yield slct.on(t) + if rows: + break + else: + log.msg("No potentially denormalized UUIDs detected, " + "skipping normalization upgrade.") + yield t.abort() + returnValue(None) + try: + yield _normalizeHomeUUIDsIn(t, ECALENDARTYPE) + yield _normalizeHomeUUIDsIn(t, EADDRESSBOOKTYPE) + yield _normalizeHomeUUIDsIn(t, ENOTIFICATIONTYPE) + yield _normalizeColumnUUIDs(t, schema.RESOURCE_PROPERTY.VIEWER_UID) + yield _normalizeColumnUUIDs(t, schema.APN_SUBSCRIPTIONS.SUBSCRIBER_GUID) + except: + log_err() + yield t.abort() + # There's a lot of possible problems here which are very hard to test + # for individually; unexpected data that might cause constraint + # violations under one of the manipulations done by + # normalizeHomeUUIDsIn. Since this upgrade does not come along with a + # schema version bump and may be re- attempted at any time, just raise + # the exception and log it so that we can try again later, and the + # service will survive for everyone _not_ affected by this somewhat + # obscure bug. + else: + yield t.commit() + + + Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql =================================================================== --- CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2012-05-24 20:40:14 UTC (rev 9297) @@ -495,5 +495,5 @@ ); insert into CALENDARSERVER values ('VERSION', '9'); -insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '2'); +insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '3'); insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '1'); Modified: CalendarServer/trunk/txdav/common/datastore/test/test_sql.py =================================================================== --- CalendarServer/trunk/txdav/common/datastore/test/test_sql.py 2012-05-24 20:34:39 UTC (rev 9296) +++ CalendarServer/trunk/txdav/common/datastore/test/test_sql.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -31,10 +31,11 @@ CALENDAR_OBJECT_REVISIONS_TABLE from txdav.common.datastore.test.util import CommonCommonTests, buildStore from txdav.common.icommondatastore import AllRetriesFailed +from twext.enterprise.dal.syntax import Insert -class SubTransactionTests(CommonCommonTests, TestCase): +class CommonSQLStoreTests(CommonCommonTests, TestCase): """ - Tests for L{UpgradeToDatabaseService}. + Tests for shared functionality in L{txdav.common.datastore.sql}. """ @inlineCallbacks @@ -42,7 +43,7 @@ """ Set up two stores to migrate between. """ - yield super(SubTransactionTests, self).setUp() + yield super(CommonSQLStoreTests, self).setUp() self._sqlStore = yield buildStore(self, self.notifierFactory) @@ -304,4 +305,33 @@ changed = yield homeChild.resourceNamesSinceToken(token) self.assertEqual(changed, ([], [],)) - txn.abort() + yield txn.abort() + + + @inlineCallbacks + def test_normalizeColumnUUIDs(self): + """ + L{_normalizeColumnUUIDs} upper-cases only UUIDs in a given column. + """ + rp = schema.RESOURCE_PROPERTY + txn = self.transactionUnderTest() + # setup + yield Insert({rp.RESOURCE_ID: 1, + rp.NAME: "asdf", + rp.VALUE: "property-value", + rp.VIEWER_UID: "not-a-uuid"}).on(txn) + yield Insert({rp.RESOURCE_ID: 2, + rp.NAME: "fdsa", + rp.VALUE: "another-value", + rp.VIEWER_UID: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"} + ).on(txn) + # test + from txdav.common.datastore.sql import _normalizeColumnUUIDs + yield _normalizeColumnUUIDs(txn, rp.VIEWER_UID) + self.assertEqual( + (yield Select([rp.RESOURCE_ID, rp.NAME, + rp.VALUE, rp.VIEWER_UID], From=rp).on(txn)), + [[1, "asdf", "property-value", "not-a-uuid"], + [2, "fdsa", "another-value", + "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"]] + ) Copied: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py (from rev 9296, CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py) =================================================================== --- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py (rev 0) +++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py 2012-05-24 20:40:14 UTC (rev 9297) @@ -0,0 +1,42 @@ +# -*- test-case-name: txdav.common.datastore.upgrade.sql.test -*- +## +# Copyright (c) 2011 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +""" +Upgrade to deal with normalization of UUIDs in +CALENDAR_HOME/ADDRESSBOOK_HOME/NOTIFICATION/APN_SUBSCRIPTIONS tables, as well +as in calendar data and properties. +""" + +from txdav.common.datastore.sql import fixUUIDNormalization +from twisted.internet.defer import inlineCallbacks +from txdav.common.datastore.upgrade.sql.upgrades.util import updateDataVersion + +UPGRADE_TO_VERSION = 3 + +@inlineCallbacks +def doUpgrade(sqlStore): + """ + Do the UUID-normalization upgrade if necessary and then bump the data + version to indicate that it's been done. + """ + yield fixUUIDNormalization(sqlStore) + + # Always bump the DB value + yield updateDataVersion( + sqlStore, "CALENDAR-DATAVERSION", UPGRADE_TO_VERSION + ) +