May 2012 - calendarserver-changes - lists.macosforge.org

[9281] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ directory/calendaruserproxy.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9281 http://trac.macosforge.org/projects/calendarserver/changeset/9281 Author: glyph(a)apple.com Date: 2012-05-24 13:33:55 -0700 (Thu, 24 May 2012) Log Message: ----------- fix up group name upgrader to be compatible with tests Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py 2012-05-24 20:33:52 UTC (rev 9280) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py 2012-05-24 20:33:55 UTC (rev 9281) @@ -800,15 +800,15 @@ ) if int(old_version) < 5: - for groupname, member in ( - (yield self.query("select GROUPNAME, MEMBER from GROUPS")) + for (groupname, member) in ( + (yield self._db_all_values_for_sql("select GROUPNAME, MEMBER from GROUPS")) ): - groupPrincipalUID, groupMode = groupname.split("#") + grouplist = groupname.split("#") + grouplist[0] = normalizeUUID(grouplist[0]) yield self._db_execute(""" - update GROUPS set (GROUPNAME = :1, MEMBER = :2) + update GROUPS set GROUPNAME = :1, MEMBER = :2 where GROUPNAME = :1 and MEMBER = :2 - """, "#".join([normalizeUUID(groupPrincipalUID), groupMode]), - normalizeUUID(member)) + """, ["#".join(grouplist), normalizeUUID(member)]) def _db_empty_data_tables(self):

1 0

[9280] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ database.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9280 http://trac.macosforge.org/projects/calendarserver/changeset/9280 Author: glyph(a)apple.com Date: 2012-05-24 13:33:52 -0700 (Thu, 24 May 2012) Log Message: ----------- oops, using adbapi, commit is implicit Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py 2012-05-24 20:33:50 UTC (rev 9279) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py 2012-05-24 20:33:52 UTC (rev 9280) @@ -349,14 +349,15 @@ # cannot be thrown away. raise NotImplementedError("Persistent databases MUST support an upgrade method.") + @inlineCallbacks def _db_upgrade_schema(self): """ Upgrade the stored schema version to the current one. """ yield self._db_execute("insert or replace into CALDAV (KEY, VALUE) values ('SCHEMA_VERSION', :1)", (self._db_version(),)) - yield self._db_commit() + @inlineCallbacks def _db_remove(self): """

1 0

[9279] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9279 http://trac.macosforge.org/projects/calendarserver/changeset/9279 Author: glyph(a)apple.com Date: 2012-05-24 13:33:50 -0700 (Thu, 24 May 2012) Log Message: ----------- dataversion upgrader that invokes the case-normalization logic Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql_schema/current.sql Added Paths: ----------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql_schema/current.sql =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql_schema/current.sql 2012-05-24 20:33:47 UTC (rev 9278) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql_schema/current.sql 2012-05-24 20:33:50 UTC (rev 9279) @@ -495,5 +495,5 @@ ); insert into CALENDARSERVER values ('VERSION', '9'); -insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '2'); +insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '3'); insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '1'); Added: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py (rev 0) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py 2012-05-24 20:33:50 UTC (rev 9279) @@ -0,0 +1,42 @@ +# -*- test-case-name: txdav.common.datastore.upgrade.sql.test -*- +## +# Copyright (c) 2011 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +""" +Upgrade to deal with normalization of UUIDs in +CALENDAR_HOME/ADDRESSBOOK_HOME/NOTIFICATION/APN_SUBSCRIPTIONS tables, as well +as in calendar data and properties. +""" + +from txdav.common.datastore.sql import fixCaseNormalization +from twisted.internet.defer import inlineCallbacks +from txdav.common.datastore.upgrade.sql.upgrades.util import updateDataVersion + +UPGRADE_TO_VERSION = 3 + +@inlineCallbacks +def doUpgrade(sqlStore): + """ + Do the required upgrade steps. + """ + + yield fixCaseNormalization(sqlStore) + + # Always bump the DB value + yield updateDataVersion( + sqlStore, "CALENDAR-DATAVERSION", UPGRADE_TO_VERSION + ) +

1 0

[9278] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ directory/augment.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9278 http://trac.macosforge.org/projects/calendarserver/changeset/9278 Author: glyph(a)apple.com Date: 2012-05-24 13:33:47 -0700 (Thu, 24 May 2012) Log Message: ----------- For now, normalize UUIDs in augments XML database whenever it's opened. Also, don't use 'OrNot' version of normalization since there are no urn:uuid:s in there. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py 2012-05-24 20:33:44 UTC (rev 9277) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py 2012-05-24 20:33:47 UTC (rev 9278) @@ -31,6 +31,7 @@ from twistedcaldav.directory.xmlaugmentsparser import XMLAugmentsParser from twistedcaldav.xmlutil import newElementTreeWithRoot, addSubElement,\ writeXML, readXML +from twistedcaldav.directory.util import normalizeUUID log = Logger() @@ -98,11 +99,10 @@ @return: a L{Deferred} that fires when all records have been normalized. """ - from txdav.base.datastore.util import normalizeUUIDOrNot remove = [] add = [] for uid in (yield self.getAllUIDs()): - nuid = normalizeUUIDOrNot(uid) + nuid = normalizeUUID(uid) if uid != nuid: old = yield self._lookupAugmentRecord(uid) new = copy.deepcopy(old) @@ -266,6 +266,7 @@ raise self.lastCached = time.time() + self.normalizeUUIDs() def getAllUIDs(self):

1 0

[9277] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ mail.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9277 http://trac.macosforge.org/projects/calendarserver/changeset/9277 Author: glyph(a)apple.com Date: 2012-05-24 13:33:44 -0700 (Thu, 24 May 2012) Log Message: ----------- note to self Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py 2012-05-24 20:33:41 UTC (rev 9276) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py 2012-05-24 20:33:44 UTC (rev 9277) @@ -771,6 +771,7 @@ update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 """, normalizeUUIDOrNot(organizer), organizer ) + # ATTENDEEs are always mailto: so unconditionally lower(). self._db_execute( """ update TOKENS set ATTENDEE = :1 WHERE ATTENDEE = :2

1 0

[9276] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ directory/calendaruserproxy.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9276 http://trac.macosforge.org/projects/calendarserver/changeset/9276 Author: glyph(a)apple.com Date: 2012-05-24 13:33:41 -0700 (Thu, 24 May 2012) Log Message: ----------- schema upgrader for proxy database to normalize possibly-UUID portions of the data Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py 2012-05-24 20:33:38 UTC (rev 9275) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/calendaruserproxy.py 2012-05-24 20:33:41 UTC (rev 9276) @@ -47,6 +47,7 @@ from twistedcaldav.directory.principal import formatLinks from twistedcaldav.directory.principal import formatPrincipals +from twistedcaldav.directory.util import normalizeUUID from twistedcaldav.config import config, fullServerPath from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\ ADBAPIPostgreSQLMixin @@ -415,11 +416,11 @@ """ - schema_version = "4" + schema_version = "5" schema_type = "CALENDARUSERPROXY" - + class ProxyDBMemcacher(Memcacher): - + def __init__(self, namespace): super(ProxyDB.ProxyDBMemcacher, self).__init__(namespace, key_normalization=config.Memcached.ProxyDBKeyNormalization) @@ -774,6 +775,7 @@ ifnotexists=True, ) + @inlineCallbacks def _db_upgrade_data_tables(self, old_version): """ @@ -797,6 +799,18 @@ ifnotexists=True, ) + if int(old_version) < 5: + for groupname, member in ( + (yield self.query("select GROUPNAME, MEMBER from GROUPS")) + ): + groupPrincipalUID, groupMode = groupname.split("#") + yield self._db_execute(""" + update GROUPS set (GROUPNAME = :1, MEMBER = :2) + where GROUPNAME = :1 and MEMBER = :2 + """, "#".join([normalizeUUID(groupPrincipalUID), groupMode]), + normalizeUUID(member)) + + def _db_empty_data_tables(self): """ Empty the underlying database tables.

1 0

[9275] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ database.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9275 http://trac.macosforge.org/projects/calendarserver/changeset/9275 Author: glyph(a)apple.com Date: 2012-05-24 13:33:38 -0700 (Thu, 24 May 2012) Log Message: ----------- add a commit after the schema upgrade Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py 2012-05-24 20:33:35 UTC (rev 9274) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/database.py 2012-05-24 20:33:38 UTC (rev 9275) @@ -355,6 +355,7 @@ Upgrade the stored schema version to the current one. """ yield self._db_execute("insert or replace into CALDAV (KEY, VALUE) values ('SCHEMA_VERSION', :1)", (self._db_version(),)) + yield self._db_commit() @inlineCallbacks def _db_remove(self):

1 0

[9274] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ directory/xmlaccountsparser.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9274 http://trac.macosforge.org/projects/calendarserver/changeset/9274 Author: glyph(a)apple.com Date: 2012-05-24 13:33:35 -0700 (Thu, 24 May 2012) Log Message: ----------- Normalize UUIDs during parsing, so that we don't need to worry about re-writing the file. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/xmlaccountsparser.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/xmlaccountsparser.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/xmlaccountsparser.py 2012-05-24 20:33:32 UTC (rev 9273) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/xmlaccountsparser.py 2012-05-24 20:33:35 UTC (rev 9274) @@ -176,15 +176,15 @@ def expand(text, ctr): """ Returns a string where ~<number> is replaced by the first <number> - characters from the md5 hexdigest of str(ctr), e.g.: + characters from the md5 hexdigest of str(ctr), e.g.:: expand("~9 foo", 1) - returns: + returns:: "c4ca4238a foo" - ...since "c4ca4238a" is the first 9 characters of: + ...since "c4ca4238a" is the first 9 characters of:: hashlib.md5(str(1)).hexdigest() @@ -258,7 +258,9 @@ self.shortNames.append(child.firstChild.data.encode("utf-8")) elif child_name == ELEMENT_GUID: if child.firstChild is not None: - self.guid = child.firstChild.data.encode("utf-8") + self.guid = normalizeUUID( + child.firstChild.data.encode("utf-8") + ) if len(self.guid) < 4: self.guid += "?" * (4 - len(self.guid)) elif child_name == ELEMENT_PASSWORD:

1 0

[9273] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ directory/augment.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9273 http://trac.macosforge.org/projects/calendarserver/changeset/9273 Author: glyph(a)apple.com Date: 2012-05-24 13:33:32 -0700 (Thu, 24 May 2012) Log Message: ----------- UUID normalizer for augment records. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py 2012-05-24 20:33:28 UTC (rev 9272) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py 2012-05-24 20:33:32 UTC (rev 9273) @@ -88,8 +88,32 @@ def __init__(self): self.cachedRecords = {} - + + @inlineCallbacks + def normalizeUUIDs(self): + """ + Normalize (uppercase) all augment UIDs which are parseable as UUIDs. + + @return: a L{Deferred} that fires when all records have been + normalized. + """ + from txdav.base.datastore.util import normalizeUUIDOrNot + remove = [] + add = [] + for uid in (yield self.getAllUIDs()): + nuid = normalizeUUIDOrNot(uid) + if uid != nuid: + old = yield self._lookupAugmentRecord(uid) + new = copy.deepcopy(old) + new.uid = uid.upper() + remove.append(old) + add.append(new) + yield self.removeAugmentRecords(remove) + yield self.addAugmentRecords(add) + + + @inlineCallbacks def getAugmentRecord(self, uid, recordType): """ Get an AugmentRecord for the specified UID or the default. @@ -244,16 +268,15 @@ self.lastCached = time.time() - @inlineCallbacks def getAllUIDs(self): """ Get all AugmentRecord UIDs. @return: L{Deferred} """ - return succeed(self.db.keys()) + def _lookupAugmentRecord(self, uid): """ Get an AugmentRecord for the specified UID.

1 0

[9272] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ mail.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9272 http://trac.macosforge.org/projects/calendarserver/changeset/9272 Author: glyph(a)apple.com Date: 2012-05-24 13:33:28 -0700 (Thu, 24 May 2012) Log Message: ----------- As long as we're always fixing case on the whole mail tokens DB at startup _anyway_, fix case for urn:uuid: as well as mailto:. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py 2012-05-24 20:33:25 UTC (rev 9271) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/mail.py 2012-05-24 20:33:28 UTC (rev 9272) @@ -744,7 +744,12 @@ ) self._db_commit() + def lowercase(self): + """ + Lowercase mailto: addresses (and uppercase urn:uuid: addresses!) so + they can be located via normalized names. + """ rows = self._db_execute( """ select ORGANIZER, ATTENDEE from TOKENS @@ -759,6 +764,13 @@ update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 """, organizer.lower(), organizer ) + else: + from txdav.base.datastore.util import normalizeUUIDOrNot + self._db_execute( + """ + update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 + """, normalizeUUIDOrNot(organizer), organizer + ) self._db_execute( """ update TOKENS set ATTENDEE = :1 WHERE ATTENDEE = :2 @@ -766,6 +778,7 @@ ) self._db_commit() + def _db_version(self): """ @return: the schema version assigned to this index.

1 0