[CalendarServer-changes] [12365] CalendarServer/trunk/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:16:47 PDT 2014
Revision: 12365
http://trac.calendarserver.org//changeset/12365
Author: gaya at apple.com
Date: 2014-01-16 15:39:26 -0800 (Thu, 16 Jan 2014)
Log Message:
-----------
use CommonStoreTransaction.calendarserverValue() & CommonStoreTransaction.updateCalendarserverValue() instead of direct ops on CALENDARSERVER table; move deleteRevisionsBefore() to CommonStoreTransaction
Modified Paths:
--------------
CalendarServer/trunk/txdav/carddav/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py
CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py
Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2014-01-16 22:22:25 UTC (rev 12364)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2014-01-16 23:39:26 UTC (rev 12365)
@@ -645,14 +645,8 @@
returnValue((yield super(AddressBook, self).resourceNamesSinceRevision(revision)))
if revision:
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self._txn))[0][0])
-
- if revision < minValidRevision:
+ minValidRevision = yield self._txn.calendarserverValue("MIN-VALID-REVISION")
+ if revision < int(minValidRevision):
raise SyncTokenValidException
# call sharedChildResourceNamesSinceRevision() and filter results
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2014-01-16 22:22:25 UTC (rev 12364)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2014-01-16 23:39:26 UTC (rev 12365)
@@ -1441,7 +1441,78 @@
return DatabaseUnlock().on(self)
+ @inlineCallbacks
+ def deleteRevisionsBefore(self, minRevision):
+ """
+ Delete revisions before minRevision
+ """
+ # Delete old revisions
+ for table in (
+ schema.CALENDAR_OBJECT_REVISIONS,
+ schema.NOTIFICATION_OBJECT_REVISIONS,
+ schema.ADDRESSBOOK_OBJECT_REVISIONS,
+ ):
+ yield Delete(
+ From=table,
+ Where=(table.REVISION < minRevision)
+ ).on(self)
+ # get groups where this object was once a member and version info
+ aboMembers = schema.ABO_MEMBERS
+ groupRows = yield Select(
+ [aboMembers.GROUP_ID,
+ aboMembers.MEMBER_ID,
+ aboMembers.REMOVED,
+ aboMembers.REVISION],
+ From=aboMembers,
+ ).on(self)
+
+ # group results by group, member, and revisionInfo
+ groupIDToMemberIDMap = {}
+ for groupRow in groupRows:
+ groupID, memberID, removed, revision = groupRow
+ revisionInfo = [removed, revision]
+ if groupID not in groupIDToMemberIDMap:
+ groupIDToMemberIDMap[groupID] = {}
+ memberIDToRevisionsMap = groupIDToMemberIDMap[groupID]
+ if memberID not in memberIDToRevisionsMap:
+ memberIDToRevisionsMap[memberID] = []
+ revisionInfoList = memberIDToRevisionsMap[memberID]
+ revisionInfoList.append(revisionInfo)
+
+ # go though list an delete old revisions, leaving at least one unremoved member
+ for groupID, memberIDToRevisionsMap in groupIDToMemberIDMap.iteritems():
+ for memberID, revisionInfoList in memberIDToRevisionsMap.iteritems():
+ revisionInfosToRemove = []
+ revisionInfosToSave = []
+ for revisionInfo in revisionInfoList:
+ if revisionInfo[1] < minRevision:
+ revisionInfosToRemove.append(revisionInfo)
+ else:
+ revisionInfosToSave.append(revisionInfo)
+
+ # save at least one revision
+ if revisionInfosToRemove and len(revisionInfosToRemove) == len(revisionInfoList):
+ maxRevisionInfoToRemove = max(revisionInfosToRemove, key=lambda info: info[1])
+ revisionInfosToSave.append(maxRevisionInfoToRemove)
+ revisionInfosToRemove.remove(maxRevisionInfoToRemove)
+
+ # get rid of extra removed member revisions
+ if revisionInfosToSave and max(revisionInfosToSave, key=lambda info: not info[0])[0]:
+ revisionInfosToRemove += revisionInfosToSave
+
+ if revisionInfosToRemove:
+ revisionsToRemove = [revisionInfoToRemove[1] for revisionInfoToRemove in revisionInfosToRemove]
+ yield Delete(
+ aboMembers,
+ Where=(aboMembers.GROUP_ID == groupID).And(
+ aboMembers.MEMBER_ID == memberID).And(
+ aboMembers.REVISION.In(Parameter("revisionsToRemove", len(revisionsToRemove)))
+ )
+ ).on(self, revisionsToRemove=revisionsToRemove)
+
+
+
class _EmptyCacher(object):
def set(self, key, value): #@UnusedVariable
@@ -2136,11 +2207,7 @@
self._syncTokenRevision = (yield self._syncTokenQuery.on(
self._txn, resourceID=self._resourceID))[0][0]
if self._syncTokenRevision is None:
- cs = schema.CALENDARSERVER
- self._syncTokenRevision = int((yield Select(
- [cs.VALUE], From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self._txn))[0][0])
+ self._syncTokenRevision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
returnValue("%s_%s" % (self._resourceID, self._syncTokenRevision))
@@ -2220,14 +2287,8 @@
deleted = set()
invalid = set()
if revision:
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self._txn))[0][0])
-
- if revision < minValidRevision:
+ minValidRevision = yield self._txn.calendarserverValue("MIN-VALID-REVISION")
+ if revision < int(minValidRevision):
raise SyncTokenValidException
results = [
@@ -2622,11 +2683,7 @@
self._syncTokenRevision = (yield self._childSyncTokenQuery.on(
self._txn, resourceID=self._resourceID))[0][0]
if self._syncTokenRevision is None:
- cs = schema.CALENDARSERVER
- self._syncTokenRevision = int((yield Select(
- [cs.VALUE], From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self._txn))[0][0])
+ self._syncTokenRevision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
returnValue(("%s_%s" % (self._resourceID, self._syncTokenRevision,)))
@@ -2674,14 +2731,8 @@
deleted = []
invalid = []
if revision:
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self._txn))[0][0])
-
- if revision < minValidRevision:
+ minValidRevision = yield self._txn.calendarserverValue("MIN-VALID-REVISION")
+ if revision < int(minValidRevision):
raise SyncTokenValidException
results = [
@@ -6053,11 +6104,7 @@
self._txn, resourceID=self._resourceID)
)[0][0]
if self._syncTokenRevision is None:
- cs = schema.CALENDARSERVER
- self._syncTokenRevision = int((yield Select(
- [cs.VALUE], From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self._txn))[0][0])
+ self._syncTokenRevision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
returnValue("%s_%s" % (self._resourceID, self._syncTokenRevision))
@@ -6789,75 +6836,3 @@
# obscure bug.
else:
yield t.commit()
-
-
-
- at inlineCallbacks
-def deleteRevisionsBefore(txn, minRevision):
- """
- Delete revisions before minRevision
- """
- # Delete old revisions
- for table in (
- schema.CALENDAR_OBJECT_REVISIONS,
- schema.NOTIFICATION_OBJECT_REVISIONS,
- schema.ADDRESSBOOK_OBJECT_REVISIONS,
- ):
- yield Delete(
- From=table,
- Where=(table.REVISION < minRevision)
- ).on(txn)
-
- # get groups where this object was once a member and version info
- aboMembers = schema.ABO_MEMBERS
- groupRows = yield Select(
- [aboMembers.GROUP_ID,
- aboMembers.MEMBER_ID,
- aboMembers.REMOVED,
- aboMembers.REVISION],
- From=aboMembers,
- ).on(txn)
-
- # group results by group, member, and revisionInfo
- groupIDToMemberIDMap = {}
- for groupRow in groupRows:
- groupID, memberID, removed, revision = groupRow
- revisionInfo = [removed, revision]
- if groupID not in groupIDToMemberIDMap:
- groupIDToMemberIDMap[groupID] = {}
- memberIDToRevisionsMap = groupIDToMemberIDMap[groupID]
- if memberID not in memberIDToRevisionsMap:
- memberIDToRevisionsMap[memberID] = []
- revisionInfoList = memberIDToRevisionsMap[memberID]
- revisionInfoList.append(revisionInfo)
-
- # go though list an delete old revisions, leaving at least one unremoved member
- for groupID, memberIDToRevisionsMap in groupIDToMemberIDMap.iteritems():
- for memberID, revisionInfoList in memberIDToRevisionsMap.iteritems():
- revisionInfosToRemove = []
- revisionInfosToSave = []
- for revisionInfo in revisionInfoList:
- if revisionInfo[1] < minRevision:
- revisionInfosToRemove.append(revisionInfo)
- else:
- revisionInfosToSave.append(revisionInfo)
-
- # save at least one revision
- if revisionInfosToRemove and len(revisionInfosToRemove) == len(revisionInfoList):
- maxRevisionInfoToRemove = max(revisionInfosToRemove, key=lambda info: info[1])
- revisionInfosToSave.append(maxRevisionInfoToRemove)
- revisionInfosToRemove.remove(maxRevisionInfoToRemove)
-
- # get rid of extra removed member revisions
- if revisionInfosToSave and max(revisionInfosToSave, key=lambda info: not info[0])[0]:
- revisionInfosToRemove += revisionInfosToSave
-
- if revisionInfosToRemove:
- revisionsToRemove = [revisionInfoToRemove[1] for revisionInfoToRemove in revisionInfosToRemove]
- yield Delete(
- aboMembers,
- Where=(aboMembers.GROUP_ID == groupID).And(
- aboMembers.MEMBER_ID == memberID).And(
- aboMembers.REVISION.In(Parameter("revisionsToRemove", len(revisionsToRemove)))
- )
- ).on(txn, revisionsToRemove=revisionsToRemove)
Modified: CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py 2014-01-16 22:22:25 UTC (rev 12364)
+++ CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py 2014-01-16 23:39:26 UTC (rev 12365)
@@ -25,7 +25,6 @@
from twext.python.log import Logger
from twisted.internet.defer import inlineCallbacks, returnValue
from twistedcaldav.config import config
-from txdav.common.datastore.sql import deleteRevisionsBefore
from txdav.common.datastore.sql_tables import schema
import datetime
@@ -53,12 +52,7 @@
yield Delete(From=self.table, Where=None).on(self.transaction)
# Get the minimum valid revision
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self.transaction))[0][0])
+ minValidRevision = int((yield self.transaction.calendarserverValue("MIN-VALID-REVISION")))
# get max revision on table rows before dateLimit
dateLimit = (datetime.datetime.utcnow() -
@@ -84,12 +78,8 @@
maxRevOlderThanDate = tableMaxRevision
if maxRevOlderThanDate > minValidRevision:
- # save it
- cs = schema.CALENDARSERVER
- yield Update(
- {cs.VALUE: maxRevOlderThanDate},
- Where=cs.NAME == "MIN-VALID-REVISION",
- ).on(self.transaction)
+ # save new min valid revision
+ yield self.transaction.updateCalendarserverValue("MIN-VALID-REVISION", maxRevOlderThanDate)
# Schedule revision cleanup
yield RevisionCleanupWork._schedule(self.transaction, seconds=0)
@@ -124,15 +114,10 @@
yield Delete(From=self.table, Where=None).on(self.transaction)
# Get the minimum valid revision
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self.transaction))[0][0])
+ minValidRevision = int((yield self.transaction.calendarserverValue("MIN-VALID-REVISION")))
# delete revisions
- yield deleteRevisionsBefore(self.transaction, minValidRevision)
+ yield self.transaction.deleteRevisionsBefore(minValidRevision)
# Schedule next update
yield FindMinValidRevisionWork._schedule(
Modified: CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py 2014-01-16 22:22:25 UTC (rev 12364)
+++ CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py 2014-01-16 23:39:26 UTC (rev 12365)
@@ -293,13 +293,8 @@
yield wp.whenExecuted()
# Get the minimum valid revision and check it
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self.transactionUnderTest()))[0][0])
- self.assertEqual(minValidRevision, max([row[0] for row in revisionRows]))
+ minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
# do RevisionCleanupWork
wp = yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
@@ -346,13 +341,8 @@
yield wp.whenExecuted()
# Get the minimum valid revision and check it
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self.transactionUnderTest()))[0][0])
- self.assertEqual(minValidRevision, max([row[0] for row in revisionRows]))
+ minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
# do RevisionCleanupWork
wp = yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
@@ -403,13 +393,8 @@
yield wp.whenExecuted()
# Get the minimum valid revision and check it
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self.transactionUnderTest()))[0][0])
- self.assertEqual(minValidRevision, max([row[0] for row in revisionRows]))
+ minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
# do RevisionCleanupWork
wp = yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
@@ -481,13 +466,8 @@
yield wp.whenExecuted()
# Get the minimum valid revision and check it
- cs = schema.CALENDARSERVER
- minValidRevision = int((yield Select(
- [cs.VALUE],
- From=cs,
- Where=(cs.NAME == "MIN-VALID-REVISION")
- ).on(self.transactionUnderTest()))[0][0])
- self.assertEqual(minValidRevision, max([row[3] for row in group1Rows + group2Rows]))
+ minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
+ self.assertEqual(int(minValidRevision), max([row[3] for row in group1Rows + group2Rows]))
# do RevisionCleanupWork
wp = yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/13b1e3c6/attachment.html>
More information about the calendarserver-changes
mailing list