[CalendarServer-changes] [10852] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Tue Mar 5 13:27:31 PST 2013 

Revision: 10852
          http://trac.calendarserver.org//changeset/10852
Author:   sagen at apple.com
Date:     2013-03-05 13:27:31 -0800 (Tue, 05 Mar 2013)
Log Message:
-----------
Landing push coalescing work from branch

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/push/notifier.py
    CalendarServer/trunk/calendarserver/push/test/test_notifier.py
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/calendarserver/tools/util.py
    CalendarServer/trunk/twext/enterprise/dal/record.py
    CalendarServer/trunk/twext/enterprise/dal/test/test_record.py
    CalendarServer/trunk/twext/enterprise/queue.py

Property Changed:
----------------
    CalendarServer/trunk/


Property changes on: CalendarServer/trunk
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/sagen/testing:10827-10830
/CalendarServer/branches/users/wsanchez/transations:5515-5593
   + /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/sagen/testing:10827-10851
/CalendarServer/branches/users/wsanchez/transations:5515-5593

Modified: CalendarServer/trunk/calendarserver/push/notifier.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/notifier.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/calendarserver/push/notifier.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -18,12 +18,13 @@
 Notification framework for Calendar Server
 """
 
+import datetime
 from twext.python.log import LoggingMixIn, Logger
-
 from twisted.internet.defer import inlineCallbacks, succeed
 from twext.enterprise.dal.record import fromTable
 from twext.enterprise.queue import WorkItem
 from txdav.common.datastore.sql_tables import schema
+from twext.enterprise.dal.syntax import Delete
 
 
 log = Logger()
@@ -31,10 +32,15 @@
 
 class PushNotificationWork(WorkItem, fromTable(schema.PUSH_NOTIFICATION_WORK)):
 
+    group = "PUSH_ID"
+
     @inlineCallbacks
     def doWork(self):
 
-        # FIXME: Coalescing goes here?
+        # Delete all other work items with the same pushID
+        yield Delete(From=self.table,
+                     Where=self.table.PUSH_ID == self.pushID 
+                    ).on(self.transaction)
 
         pushDistributor = self.transaction._pushDistributor
         if pushDistributor is not None:
@@ -117,9 +123,10 @@
     work queue.
     """
 
-    def __init__(self, store, hostname, reactor=None):
+    def __init__(self, store, hostname, coalesceSeconds, reactor=None):
         self.store = store
         self.hostname = hostname
+        self.coalesceSeconds = coalesceSeconds
 
         if reactor is None:
             from twisted.internet import reactor
@@ -128,7 +135,10 @@
     @inlineCallbacks
     def send(self, id):
         txn = self.store.newTransaction()
-        yield txn.enqueue(PushNotificationWork, pushID=self.pushKeyForId(id))
+        notBefore = datetime.datetime.utcnow() + datetime.timedelta(
+            seconds=self.coalesceSeconds)
+        yield txn.enqueue(PushNotificationWork, pushID=self.pushKeyForId(id),
+            notBefore=notBefore)
         yield txn.commit()
 
     def newNotifier(self, label="default", id=None, prefix=None):

Modified: CalendarServer/trunk/calendarserver/push/test/test_notifier.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/test/test_notifier.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/calendarserver/push/test/test_notifier.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -15,10 +15,14 @@
 ##
 
 from twistedcaldav.test.util import TestCase
-from calendarserver.push.notifier import PushDistributor, getPubSubAPSConfiguration
+from calendarserver.push.notifier import PushDistributor
+from calendarserver.push.notifier import getPubSubAPSConfiguration
+from calendarserver.push.notifier import PushNotificationWork
 from twisted.internet.defer import inlineCallbacks, succeed
 from twistedcaldav.config import ConfigDict
+from txdav.common.datastore.test.util import buildStore
 
+
 class StubService(object):
     def __init__(self):
         self.reset()
@@ -69,3 +73,50 @@
                 "APSEnvironment": "prod"
             }
         )
+
+
+class StubDistributor(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.history = []
+
+    def enqueue(self, pushID):
+        self.history.append(pushID)
+
+class PushNotificationWorkTests(TestCase):
+
+    @inlineCallbacks
+    def test_work(self):
+        self.store = yield buildStore(self, None)
+
+        pushDistributor = StubDistributor()
+
+        def decorateTransaction(txn):
+            txn._pushDistributor = pushDistributor
+
+        self.store.callWithNewTransactions(decorateTransaction)
+
+        txn = self.store.newTransaction()
+        wp = (yield txn.enqueue(PushNotificationWork,
+            pushID="/CalDAV/localhost/foo/",
+        ))
+        yield txn.commit()
+        yield wp.whenExecuted()
+        self.assertEquals(pushDistributor.history, ["/CalDAV/localhost/foo/"])
+
+        pushDistributor.reset()
+        txn = self.store.newTransaction()
+        wp = (yield txn.enqueue(PushNotificationWork,
+            pushID="/CalDAV/localhost/bar/",
+        ))
+        wp = (yield txn.enqueue(PushNotificationWork,
+            pushID="/CalDAV/localhost/bar/",
+        ))
+        wp = (yield txn.enqueue(PushNotificationWork,
+            pushID="/CalDAV/localhost/bar/",
+        ))
+        yield txn.commit()
+        yield wp.whenExecuted()
+        self.assertEquals(pushDistributor.history, ["/CalDAV/localhost/bar/"])

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -225,7 +225,8 @@
     if config.Notifications.Enabled:
         # FIXME: NotifierFactory needs reference to the store in order
         # to get a txn in order to create a Work item
-        notifierFactory = NotifierFactory(None, config.ServerHostName)
+        notifierFactory = NotifierFactory(None, config.ServerHostName,
+            config.Notifications.CoalesceSeconds)
     else:
         notifierFactory = None
     quota = config.UserQuota

Modified: CalendarServer/trunk/calendarserver/tools/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/util.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/calendarserver/tools/util.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -81,6 +81,7 @@
                     # to get a txn in order to create a Work item
                     notifierFactory = NotifierFactory(
                         None, config.ServerHostName,
+                        config.Notifications.CoalesceSeconds,
                     )
                 else:
                     notifierFactory = None

Modified: CalendarServer/trunk/twext/enterprise/dal/record.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/dal/record.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/twext/enterprise/dal/record.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -190,9 +190,12 @@
     @classmethod
     @inlineCallbacks
     def load(cls, transaction, *primaryKey):
-        self = (yield cls.query(transaction,
-                                cls._primaryKeyComparison(primaryKey)))[0]
-        returnValue(self)
+        results = (yield cls.query(transaction,
+                                cls._primaryKeyComparison(primaryKey)))
+        if len(results) != 1:
+            raise NoSuchRecord()
+        else:
+            returnValue(results[0])
 
 
     @classmethod

Modified: CalendarServer/trunk/twext/enterprise/dal/test/test_record.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/dal/test/test_record.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/twext/enterprise/dal/test/test_record.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -98,6 +98,16 @@
 
 
     @inlineCallbacks
+    def test_missingLoad(self):
+        """
+        Try loading an row which doesn't exist
+        """
+        txn = self.pool.connection()
+        yield txn.execSQL("insert into ALPHA values (:1, :2)", [234, "one"])
+        self.assertFailure(TestRecord.load(txn, 456), NoSuchRecord)
+
+
+    @inlineCallbacks
     def test_simpleCreate(self):
         """
         When a record object is created, a row with matching column values will

Modified: CalendarServer/trunk/twext/enterprise/queue.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/queue.py	2013-03-05 21:12:03 UTC (rev 10851)
+++ CalendarServer/trunk/twext/enterprise/queue.py	2013-03-05 21:27:31 UTC (rev 10852)
@@ -97,7 +97,7 @@
 from twext.enterprise.dal.syntax import SchemaSyntax, Lock, NamedValue
 
 from twext.enterprise.dal.model import ProcedureCall
-from twext.enterprise.dal.record import Record, fromTable
+from twext.enterprise.dal.record import Record, fromTable, NoSuchRecord
 from twisted.python.failure import Failure
 
 from twext.enterprise.dal.model import Table, Schema, SQLType, Constraint
@@ -826,15 +826,19 @@
     @inlineCallbacks
     def work(txn):
         workItemClass = WorkItem.forTable(table)
-        workItem = yield workItemClass.load(txn, workID)
-        if workItem.group is not None:
-            yield NamedLock.acquire(txn, workItem.group)
-        # TODO: what if we fail?  error-handling should be recorded someplace,
-        # the row should probably be marked, re-tries should be triggerable
-        # administratively.
-        yield workItem.delete()
-        # TODO: verify that workID is the primary key someplace.
-        yield workItem.doWork()
+        try:
+            workItem = yield workItemClass.load(txn, workID)
+            if workItem.group is not None:
+                yield NamedLock.acquire(txn, workItem.group)
+            # TODO: what if we fail?  error-handling should be recorded someplace,
+            # the row should probably be marked, re-tries should be triggerable
+            # administratively.
+            yield workItem.delete()
+            # TODO: verify that workID is the primary key someplace.
+            yield workItem.doWork()
+        except NoSuchRecord:
+            # The record has already been removed
+            pass
     return inTransaction(txnFactory, work)
 
 
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130305/1f1509a4/attachment-0001.html>

More information about the calendarserver-changes mailing list