May 2012 - calendarserver-changes - lists.macosforge.org

[9291] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9291 http://trac.macosforge.org/projects/calendarserver/changeset/9291 Author: glyph(a)apple.com Date: 2012-05-24 13:34:24 -0700 (Thu, 24 May 2012) Log Message: ----------- Skip the upgrade if it's immediately obvious you can do so. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:21 UTC (rev 9290) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:24 UTC (rev 9291) @@ -67,15 +67,11 @@ from twext.python.clsprop import classproperty from twext.enterprise.ienterprise import AlreadyFinishedError from twext.enterprise.dal.parseschema import significant -from twext.enterprise.dal.syntax import Delete, utcNowSQL, Union -from twext.enterprise.dal.syntax import Insert -from twext.enterprise.dal.syntax import Len -from twext.enterprise.dal.syntax import Max -from twext.enterprise.dal.syntax import Parameter -from twext.enterprise.dal.syntax import SavepointAction -from twext.enterprise.dal.syntax import Select -from twext.enterprise.dal.syntax import Update +from twext.enterprise.dal.syntax import \ + Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction, \ + Select, Update, ColumnSyntax, TableSyntax, Upper + from txdav.base.propertystore.base import PropertyName from txdav.base.propertystore.none import PropertyStore as NonePropertyStore from txdav.base.propertystore.sql import PropertyStore @@ -87,8 +83,6 @@ from txdav.xml.rfc2518 import DisplayName from txdav.base.datastore.util import normalizeUUIDOrNot -from twext.enterprise.dal.syntax import ColumnSyntax -from twext.enterprise.dal.syntax import TableSyntax from cStringIO import StringIO from sqlparse import parse @@ -4205,11 +4199,27 @@ @inlineCallbacks -def fixCaseNormalization(store): +def fixUUIDNormalization(store): """ - Fix all case normalization for a given store. + Fix all UUIDs in the given SQL store to be in a canonical form; + 00000000-0000-0000-0000-000000000000 format and upper-case. """ t = store.newTransaction(disableCache=True) + + # First, let's see if there are any calendar or addressbook homes that have + # a lower-case OWNER_UID. If there are none, then we can early-out and + # avoid the tedious and potentially expensive inspection of oodles of + # calendar data. + for x in [schema.CALENDAR_HOME, schema.ADDRESSBOOK_HOME]: + slct = Select([x.OWNER_UID], From=x, + Where=x.OWNER_UID != Upper(x.OWNER_UID)) + rows = yield slct.on(t) + if rows: + break + else: + log.msg("No potentially denormalized UUIDs detected, " + "skipping normalization upgrade.") + returnValue(None) try: yield _normalizeHomeUUIDsIn(t, ECALENDARTYPE) yield _normalizeHomeUUIDsIn(t, EADDRESSBOOKTYPE) Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py 2012-05-24 20:34:21 UTC (rev 9290) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py 2012-05-24 20:34:24 UTC (rev 9291) @@ -21,7 +21,7 @@ as in calendar data and properties. """ -from txdav.common.datastore.sql import fixCaseNormalization +from txdav.common.datastore.sql import fixUUIDNormalization from twisted.internet.defer import inlineCallbacks from txdav.common.datastore.upgrade.sql.upgrades.util import updateDataVersion @@ -30,10 +30,10 @@ @inlineCallbacks def doUpgrade(sqlStore): """ - Do the case-normalization upgrade if necessary and then bump the data + Do the UUID-normalization upgrade if necessary and then bump the data version to indicate that it's been done. """ - yield fixCaseNormalization(sqlStore) + yield fixUUIDNormalization(sqlStore) # Always bump the DB value yield updateDataVersion(

1 0

[9290] CalendarServer/branches/users/glyph/uuid-normalize/txdav
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9290 http://trac.macosforge.org/projects/calendarserver/changeset/9290 Author: glyph(a)apple.com Date: 2012-05-24 13:34:21 -0700 (Thu, 24 May 2012) Log Message: ----------- Disable caching entirely during this transaction, since proper cache invalidation after 'update...OWNER_UID...' is almost impossible. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/sql.py CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/sql.py 2012-05-24 20:34:18 UTC (rev 9289) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/sql.py 2012-05-24 20:34:21 UTC (rev 9290) @@ -508,7 +508,7 @@ ).on(self._txn) self._supportedComponents = supported_components - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher is not None: cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID) yield queryCacher.invalidateAfterCommit(self._txn, cacheKey) Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:18 UTC (rev 9289) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:21 UTC (rev 9290) @@ -187,7 +187,7 @@ return [] - def newTransaction(self, label="unlabeled"): + def newTransaction(self, label="unlabeled", disableCache=False): """ @see: L{IDataStore.newTransaction} """ @@ -199,12 +199,14 @@ self.notifierFactory if self._enableNotifications else None, label, self._migrating, + disableCache ) - if self.logTransactionWaits or self.timeoutTransactions: - CommonStoreTransactionMonitor(txn, self.logTransactionWaits, self.timeoutTransactions) + CommonStoreTransactionMonitor(txn, self.logTransactionWaits, + self.timeoutTransactions) return txn + def setMigrating(self, state): """ Set the "migrating" state @@ -312,7 +314,7 @@ def __init__(self, store, sqlTxn, enableCalendars, enableAddressBooks, - notifierFactory, label, migrating=False): + notifierFactory, label, migrating=False, disableCache=False): self._store = store self._calendarHomes = {} self._addressbookHomes = {} @@ -324,6 +326,11 @@ self._label = label self._migrating = migrating self._primaryHomeType = None + self._disableCache = disableCache + if disableCache: + self._queryCacher = None + else: + self._queryCacher = store.queryCacher CommonStoreTransaction.id += 1 self._txid = CommonStoreTransaction.id @@ -846,6 +853,15 @@ returnValue(count) +class _EmptyCacher(object): + def set(self, key, value): + return succeed(True) + def get(self, key, withIdentifier=False): + return succeed(None) + def delete(self, key): + return succeed(True) + + class CommonHome(LoggingMixIn): # All these need to be initialized by derived classes for each store type @@ -877,6 +893,8 @@ self._created = None self._modified = None self._syncTokenRevision = None + if transaction._disableCache: + self._cacher = _EmptyCacher() # Needed for REVISION/BIND table join self._revisionBindJoinTable = {} @@ -937,7 +955,7 @@ if result: self._resourceID = result[0][0] - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher: # Get cached copy cacheKey = queryCacher.keyForHomeMetaData(self._resourceID) @@ -1519,7 +1537,7 @@ try: self._modified = (yield self._txn.subtransaction(_bumpModified, retries=0, failureOK=True))[0][0] - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher is not None: cacheKey = queryCacher.keyForHomeMetaData(self._resourceID) yield queryCacher.invalidateAfterCommit(self._txn, cacheKey) @@ -2285,7 +2303,7 @@ exists. """ data = None - queryCacher = home._txn.store().queryCacher + queryCacher = home._txn._queryCacher # Only caching non-shared objects so that we don't need to invalidate # in sql_legacy if owned and queryCacher: @@ -2447,7 +2465,7 @@ resource ID. We read in and cache all the extra metadata from the DB to avoid having to do DB queries for those individually later. """ - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher: # Retrieve from cache cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID) @@ -2521,7 +2539,7 @@ """ oldName = self._name - queryCacher = self._home._txn.store().queryCacher + queryCacher = self._home._txn._queryCacher if queryCacher: cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, oldName) yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey) @@ -2554,7 +2572,7 @@ @inlineCallbacks def remove(self): - queryCacher = self._home._txn.store().queryCacher + queryCacher = self._home._txn._queryCacher if queryCacher: cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name) yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey) @@ -2981,7 +2999,7 @@ try: self._modified = (yield self._txn.subtransaction(_bumpModified, retries=0, failureOK=True))[0][0] - queryCacher = self._txn.store().queryCacher + queryCacher = self._txn._queryCacher if queryCacher is not None: cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID) yield queryCacher.invalidateAfterCommit(self._txn, cacheKey) @@ -4191,7 +4209,7 @@ """ Fix all case normalization for a given store. """ - t = store.newTransaction() + t = store.newTransaction(disableCache=True) try: yield _normalizeHomeUUIDsIn(t, ECALENDARTYPE) yield _normalizeHomeUUIDsIn(t, EADDRESSBOOKTYPE)

1 0

[9289] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore/sql.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9289 http://trac.macosforge.org/projects/calendarserver/changeset/9289 Author: glyph(a)apple.com Date: 2012-05-24 13:34:18 -0700 (Thu, 24 May 2012) Log Message: ----------- More comprehensive support for notifications. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:15 UTC (rev 9288) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:18 UTC (rev 9289) @@ -3964,7 +3964,7 @@ column which is the maximum value. @rtype: L{Select} """ - if type == ENOTIFICATIONTYPE: + if homeType == ENOTIFICATIONTYPE: return Select( [Max(schema.NOTIFICATION.MODIFIED)], From=schema.NOTIFICATION_HOME.join( @@ -4022,17 +4022,21 @@ from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome from txdav.carddav.datastore.util import migrateHome as migrateABHome migrateHome = {EADDRESSBOOKTYPE: migrateABHome, - ECALENDARTYPE: migrateCalendarHome}[homeType] + ECALENDARTYPE: migrateCalendarHome, + ENOTIFICATIONTYPE: _dontBotherWithNotifications}[homeType] homeTable = {EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME, - ECALENDARTYPE: schema.CALENDAR_HOME}[homeType] + ECALENDARTYPE: schema.CALENDAR_HOME, + ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME}[homeType] both = [] - both.append([one, (yield determineNewest(one.uid(), type).on(sqlTxn))]) - both.append([other, (yield determineNewest(other.uid(), type).on(sqlTxn))]) + both.append([one, + (yield determineNewest(one.uid(), homeType).on(sqlTxn))]) + both.append([other, + (yield determineNewest(other.uid(), homeType).on(sqlTxn))]) both.sort(key=lambda x: x[1]) # Note: determineNewest may return None sometimes. older = both[0][0] newer = both[1][0] - yield migrateHome(older, newer, True) + yield migrateHome(older, newer, merge=True) # Rename the old one to 'old.<correct-guid>' yield Update({homeTable.OWNER_UID: "old." + older.uid().upper()}, Where=homeTable.OWNER_UID == older.uid()).on(sqlTxn) @@ -4046,6 +4050,14 @@ +def _dontBotherWithNotifications(older, newer, merge): + """ + Notifications are more transient and can be easily worked around; don't + bother to migrate all of them when there is a UUID case mismatch. + """ + + + @inlineCallbacks def _normalizeHomeUUIDsIn(t, homeType): """ @@ -4067,6 +4079,7 @@ homeTable = {EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME, ECALENDARTYPE: schema.CALENDAR_HOME, ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME}[homeType] + homeTypeName = homeTable.model.name.split("_")[0] allUIDs = yield Select([homeTable.OWNER_UID], From=homeTable, @@ -4081,12 +4094,16 @@ else: estimate = "unknown" log_msg( - format="Scanning UID %(uid)s " + format="Scanning UID %(uid)s [%(homeType)s] " "(%(pct)0.2d%%, %(estimate)s seconds remaining)...", uid=UID, pct=(n / float(total)) * 100, estimate=estimate, + homeType=homeTypeName ) other = None - this = yield t.homeWithUID(UID) + if homeType == ENOTIFICATIONTYPE: + this = yield t.notificationsWithUID(UID) + else: + this = yield t.homeWithUID(homeType, UID) if homeType == ECALENDARTYPE: fixedThisHome = yield fixOneCalendarHome(this) else: @@ -4101,15 +4118,16 @@ else: newname = str(uuidobj).upper() if UID != newname: - log_msg(format="Detected case variance: %(uid)s %(newuid)s", - uid=UID, newuid=newname) + log_msg(format="Detected case variance: %(uid)s %(newuid)s" + "[%(homeType)s]", + uid=UID, newuid=newname, homeType=homeTypeName) other = yield t.homeWithUID(homeType, newname) if other is not None: this = yield mergeHomes(t, this, other, homeType) # NOTE: WE MUST NOT TOUCH EITHER HOME OBJECT AFTER THIS # POINT. THE UIDS HAVE CHANGED AND ALL OPERATIONS WILL # FAIL. - # else: case - it's already been updated. + end = time.time() elapsed = end - start allElapsed.append(elapsed)

1 0

[9288] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore/sql.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9288 http://trac.macosforge.org/projects/calendarserver/changeset/9288 Author: glyph(a)apple.com Date: 2012-05-24 13:34:15 -0700 (Thu, 24 May 2012) Log Message: ----------- include notification home in upgrader too Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:12 UTC (rev 9287) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:15 UTC (rev 9288) @@ -4177,6 +4177,7 @@ try: yield _normalizeHomeUUIDsIn(t, ECALENDARTYPE) yield _normalizeHomeUUIDsIn(t, EADDRESSBOOKTYPE) + yield _normalizeHomeUUIDsIn(t, ENOTIFICATIONTYPE) yield _upcaseColumnUUIDs(t, schema.RESOURCE_PROPERTY.VIEWER_UID) yield _upcaseColumnUUIDs(t, schema.APN_SUBSCRIPTIONS.SUBSCRIBER_GUID) except:

1 0

[9287] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore/sql.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9287 http://trac.macosforge.org/projects/calendarserver/changeset/9287 Author: glyph(a)apple.com Date: 2012-05-24 13:34:12 -0700 (Thu, 24 May 2012) Log Message: ----------- include notification home Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:09 UTC (rev 9286) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:12 UTC (rev 9287) @@ -4056,8 +4056,8 @@ @param t: the transaction to normalize all the UUIDs in. @type t: L{CommonStoreTransaction} - @param homeType: The type of home to scan; ECALENDARTYPE or - EADDRESSBOOKTYPE. + @param homeType: The type of home to scan, L{ECALENDARTYPE}, + L{EADDRESSBOOKTYPE}, or L{ENOTIFICATIONTYPE}. @type homeType: C{int} @return: a L{Deferred} which fires with C{None} when the UUID normalization @@ -4065,7 +4065,8 @@ """ from txdav.caldav.datastore.util import fixOneCalendarHome homeTable = {EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME, - ECALENDARTYPE: schema.CALENDAR_HOME}[homeType] + ECALENDARTYPE: schema.CALENDAR_HOME, + ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME}[homeType] allUIDs = yield Select([homeTable.OWNER_UID], From=homeTable, @@ -4130,10 +4131,12 @@ @param column: the column to uppercase. Note @type column: L{ColumnSyntax} - @return: A L{Deferred} that will fire when the upper-casing of the given - column has completed. + @return: A L{Deferred} that will fire when the UUID normalization of the + given column has completed. """ tableModel = column.model.table + # Get a primary key made of column syntax objects for querying and + # comparison later. pkey = [ColumnSyntax(columnModel) for columnModel in tableModel.primaryKey] for row in (yield Select([column] + pkey, @@ -4143,6 +4146,8 @@ after = normalizeUUIDOrNot(before) if after != before: where = _AndNothing + # Build a where clause out of the primary key and the parts of the + # primary key that were found. for pkeycol, pkeypart in zip(pkeyparts, pkey): where = where.And(pkeycol == pkeypart) yield Update({column: after}, Where=where).on(txn)

1 0

[9286] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9286 http://trac.macosforge.org/projects/calendarserver/changeset/9286 Author: glyph(a)apple.com Date: 2012-05-24 13:34:09 -0700 (Thu, 24 May 2012) Log Message: ----------- Fix and test for UUID normalization on specific columns rather than dumb uppercasing. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/test/test_sql.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:06 UTC (rev 9285) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:09 UTC (rev 9286) @@ -85,9 +85,11 @@ from twistedcaldav.dateops import datetimeMktime, parseSQLTimestamp,\ pyCalendarTodatetime from txdav.xml.rfc2518 import DisplayName -from twext.enterprise.dal.syntax import Upper -from twext.enterprise.dal.syntax import Constant +from txdav.base.datastore.util import normalizeUUIDOrNot +from twext.enterprise.dal.syntax import ColumnSyntax +from twext.enterprise.dal.syntax import TableSyntax + from cStringIO import StringIO from sqlparse import parse import collections @@ -4117,20 +4119,50 @@ -def _upcaseColumn(column): +@inlineCallbacks +def _upcaseColumnUUIDs(txn, column): """ - Generate query to upper-case a single column. + Upper-case the UUIDs in the given SQL DAL column. - @param column: the column to uppercase + @param txn: The transaction. + @type txn: L{CommonStoreTransaction} + + @param column: the column to uppercase. Note @type column: L{ColumnSyntax} - @return: a query that, when executed, will cause the given column to become - upper-case according to the database's C{UPPER} function. + @return: A L{Deferred} that will fire when the upper-casing of the given + column has completed. """ - return Update({column: Upper(column)}, Where=Constant(True)) + tableModel = column.model.table + pkey = [ColumnSyntax(columnModel) + for columnModel in tableModel.primaryKey] + for row in (yield Select([column] + pkey, + From=TableSyntax(tableModel)).on(txn)): + before = row[0] + pkeyparts = row[1:] + after = normalizeUUIDOrNot(before) + if after != before: + where = _AndNothing + for pkeycol, pkeypart in zip(pkeyparts, pkey): + where = where.And(pkeycol == pkeypart) + yield Update({column: after}, Where=where).on(txn) +class _AndNothing(object): + """ + Simple placeholder for iteratively generating a 'Where' clause; the 'And' + just returns its argument, so it can be used at the start of the loop. + """ + @staticmethod + def And(self): + """ + Return the argument. + """ + return self + + + @inlineCallbacks def fixCaseNormalization(store): """ @@ -4140,8 +4172,8 @@ try: yield _normalizeHomeUUIDsIn(t, ECALENDARTYPE) yield _normalizeHomeUUIDsIn(t, EADDRESSBOOKTYPE) - yield _upcaseColumn(schema.RESOURCE_PROPERTY.VIEWER_UID).on(t) - yield _upcaseColumn(schema.APN_SUBSCRIPTIONS.SUBSCRIBER_GUID).on(t) + yield _upcaseColumnUUIDs(t, schema.RESOURCE_PROPERTY.VIEWER_UID) + yield _upcaseColumnUUIDs(t, schema.APN_SUBSCRIPTIONS.SUBSCRIBER_GUID) except: log_err() yield t.abort() Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/test/test_sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/test/test_sql.py 2012-05-24 20:34:06 UTC (rev 9285) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/test/test_sql.py 2012-05-24 20:34:09 UTC (rev 9286) @@ -31,10 +31,11 @@ CALENDAR_OBJECT_REVISIONS_TABLE from txdav.common.datastore.test.util import CommonCommonTests, buildStore from txdav.common.icommondatastore import AllRetriesFailed +from twext.enterprise.dal.syntax import Insert -class SubTransactionTests(CommonCommonTests, TestCase): +class CommonSQLStoreTests(CommonCommonTests, TestCase): """ - Tests for L{UpgradeToDatabaseService}. + Tests for shared functionality in L{txdav.common.datastore.sql}. """ @inlineCallbacks @@ -42,7 +43,7 @@ """ Set up two stores to migrate between. """ - yield super(SubTransactionTests, self).setUp() + yield super(CommonSQLStoreTests, self).setUp() self._sqlStore = yield buildStore(self, self.notifierFactory) @@ -304,4 +305,33 @@ changed = yield homeChild.resourceNamesSinceToken(token) self.assertEqual(changed, ([], [],)) - txn.abort() + yield txn.abort() + + + @inlineCallbacks + def test_upcaseColumnUUIDs(self): + """ + L{_upcaseColumnUUIDs} upper-cases only UUIDs in a given column. + """ + rp = schema.RESOURCE_PROPERTY + txn = self.transactionUnderTest() + # setup + yield Insert({rp.RESOURCE_ID: 1, + rp.NAME: "asdf", + rp.VALUE: "property-value", + rp.VIEWER_UID: "not-a-uuid"}).on(txn) + yield Insert({rp.RESOURCE_ID: 2, + rp.NAME: "fdsa", + rp.VALUE: "another-value", + rp.VIEWER_UID: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"} + ).on(txn) + # test + from txdav.common.datastore.sql import _upcaseColumnUUIDs + yield _upcaseColumnUUIDs(txn, rp.VIEWER_UID) + self.assertEqual( + (yield Select([rp.RESOURCE_ID, rp.NAME, + rp.VALUE, rp.VIEWER_UID], From=rp).on(txn)), + [[1, "asdf", "property-value", "not-a-uuid"], + [2, "fdsa", "another-value", + "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"]] + )

1 0

[9285] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9285 http://trac.macosforge.org/projects/calendarserver/changeset/9285 Author: glyph(a)apple.com Date: 2012-05-24 13:34:06 -0700 (Thu, 24 May 2012) Log Message: ----------- Better docstring. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py 2012-05-24 20:34:04 UTC (rev 9284) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/upgrade/sql/upgrades/upgrade_from_2_to_3.py 2012-05-24 20:34:06 UTC (rev 9285) @@ -30,9 +30,9 @@ @inlineCallbacks def doUpgrade(sqlStore): """ - Do the required upgrade steps. + Do the case-normalization upgrade if necessary and then bump the data + version to indicate that it's been done. """ - yield fixCaseNormalization(sqlStore) # Always bump the DB value

1 0

[9284] CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/ datastore/sql.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9284 http://trac.macosforge.org/projects/calendarserver/changeset/9284 Author: glyph(a)apple.com Date: 2012-05-24 13:34:04 -0700 (Thu, 24 May 2012) Log Message: ----------- fix logging conventions Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:01 UTC (rev 9283) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/common/datastore/sql.py 2012-05-24 20:34:04 UTC (rev 9284) @@ -30,6 +30,7 @@ from zope.interface import implements, directlyProvides from twext.python.log import Logger, LoggingMixIn +from twisted.python.log import msg as log_msg, err as log_err from txdav.xml.rfc2518 import ResourceType from txdav.xml.parser import WebDAVDocument from twext.web2.http_headers import MimeType @@ -4076,7 +4077,7 @@ total - n) else: estimate = "unknown" - log.msg( + log_msg( format="Scanning UID %(uid)s " "(%(pct)0.2d%%, %(estimate)s seconds remaining)...", uid=UID, pct=(n / float(total)) * 100, estimate=estimate, @@ -4088,7 +4089,7 @@ else: fixedThisHome = 0 if this is None: - log.msg(format="%(uid)r appears to be missing, already processed", + log_msg(format="%(uid)r appears to be missing, already processed", uid=UID) try: uuidobj = UUID(UID) @@ -4097,7 +4098,7 @@ else: newname = str(uuidobj).upper() if UID != newname: - log.msg(format="Detected case variance: %(uid)s %(newuid)s", + log_msg(format="Detected case variance: %(uid)s %(newuid)s", uid=UID, newuid=newname) other = yield t.homeWithUID(homeType, newname) if other is not None: @@ -4109,7 +4110,7 @@ end = time.time() elapsed = end - start allElapsed.append(elapsed) - log.msg(format="Scanned UID %(uid)s; %(elapsed)s seconds elapsed," + log_msg(format="Scanned UID %(uid)s; %(elapsed)s seconds elapsed," " %(fixes)s properties fixed.", uid=UID, elapsed=elapsed, fixes=fixedThisHome) returnValue(None) @@ -4137,12 +4138,12 @@ """ t = store.newTransaction() try: - yield _normalizeHomeUUIDsIn(store, ECALENDARTYPE) - yield _normalizeHomeUUIDsIn(store, EADDRESSBOOKTYPE) + yield _normalizeHomeUUIDsIn(t, ECALENDARTYPE) + yield _normalizeHomeUUIDsIn(t, EADDRESSBOOKTYPE) yield _upcaseColumn(schema.RESOURCE_PROPERTY.VIEWER_UID).on(t) yield _upcaseColumn(schema.APN_SUBSCRIPTIONS.SUBSCRIBER_GUID).on(t) except: - log.err() + log_err() yield t.abort() # There's a lot of possible problems here which are very hard to test # for individually; unexpected data that might cause constraint

1 0

[9283] CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/ datastore/util.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9283 http://trac.macosforge.org/projects/calendarserver/changeset/9283 Author: glyph(a)apple.com Date: 2012-05-24 13:34:01 -0700 (Thu, 24 May 2012) Log Message: ----------- That variable is used later too. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/util.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/util.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/util.py 2012-05-24 20:33:58 UTC (rev 9282) +++ CalendarServer/branches/users/glyph/uuid-normalize/txdav/caldav/datastore/util.py 2012-05-24 20:34:01 UTC (rev 9283) @@ -20,7 +20,6 @@ """ import os -from uuid import UUID from zope.interface.declarations import implements @@ -338,8 +337,8 @@ @return: a L{Deferred} that fires with C{None} when the migration is complete. """ + from twistedcaldav.config import config if not merge: - from twistedcaldav.config import config yield outHome.removeCalendarWithName("calendar") if config.RestrictCalendarsToOneComponentType: yield outHome.removeCalendarWithName("tasks")

1 0

[9282] CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/ directory/augment.py
by source_changes＠macosforge.org 24 May '12

24 May '12

Revision: 9282 http://trac.macosforge.org/projects/calendarserver/changeset/9282 Author: glyph(a)apple.com Date: 2012-05-24 13:33:58 -0700 (Thu, 24 May 2012) Log Message: ----------- It's OK if we can't rewrite the augments file. Modified Paths: -------------- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py Modified: CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py =================================================================== --- CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py 2012-05-24 20:33:55 UTC (rev 9281) +++ CalendarServer/branches/users/glyph/uuid-normalize/twistedcaldav/directory/augment.py 2012-05-24 20:33:58 UTC (rev 9282) @@ -109,8 +109,12 @@ new.uid = uid.upper() remove.append(old) add.append(new) - yield self.removeAugmentRecords(remove) - yield self.addAugmentRecords(add) + try: + yield self.removeAugmentRecords(remove) + yield self.addAugmentRecords(add) + except IOError: + # It's OK if we can't re-write the file. + pass @inlineCallbacks

1 0