[CalendarServer-changes] [10748] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Fri Feb 15 16:11:19 PST 2013


Revision: 10748
          http://trac.calendarserver.org//changeset/10748
Author:   sagen at apple.com
Date:     2013-02-15 16:11:19 -0800 (Fri, 15 Feb 2013)
Log Message:
-----------
Using the work queue for push and imip

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/provision/root.py
    CalendarServer/trunk/calendarserver/provision/test/test_root.py
    CalendarServer/trunk/calendarserver/push/amppush.py
    CalendarServer/trunk/calendarserver/push/applepush.py
    CalendarServer/trunk/calendarserver/push/test/test_amppush.py
    CalendarServer/trunk/calendarserver/push/test/test_applepush.py
    CalendarServer/trunk/calendarserver/tap/caldav.py
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/calendarserver/tools/changeip_calendar.py
    CalendarServer/trunk/calendarserver/tools/test/test_changeip.py
    CalendarServer/trunk/calendarserver/tools/util.py
    CalendarServer/trunk/contrib/performance/benchmarks/find_calendars.py
    CalendarServer/trunk/twext/enterprise/ienterprise.py
    CalendarServer/trunk/twext/enterprise/queue.py
    CalendarServer/trunk/twext/enterprise/test/test_queue.py
    CalendarServer/trunk/twisted/plugins/caldav.py
    CalendarServer/trunk/twistedcaldav/customxml.py
    CalendarServer/trunk/twistedcaldav/resource.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/delivery.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/mailgateway.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/scheduler.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_mailgateway.py
    CalendarServer/trunk/twistedcaldav/scheduling/ischedule/resource.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/twistedcaldav/test/test_config.py
    CalendarServer/trunk/twistedcaldav/test/test_resource.py
    CalendarServer/trunk/twistedcaldav/upgrade.py
    CalendarServer/trunk/txdav/caldav/datastore/test/common.py
    CalendarServer/trunk/txdav/carddav/datastore/test/common.py
    CalendarServer/trunk/txdav/common/datastore/file.py
    CalendarServer/trunk/txdav/common/datastore/sql.py
    CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
    CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql
    CalendarServer/trunk/txdav/common/datastore/test/util.py
    CalendarServer/trunk/txdav/common/icommondatastore.py

Added Paths:
-----------
    CalendarServer/trunk/calendarserver/push/notifier.py
    CalendarServer/trunk/calendarserver/push/test/test_notifier.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/inbound.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/outbound.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/smtpsender.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_inbound.py
    CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_outbound.py
    CalendarServer/trunk/txdav/common/datastore/sql_schema/ReadMe.txt
    CalendarServer/trunk/txdav/common/datastore/sql_schema/old/oracle-dialect/v15.sql
    CalendarServer/trunk/txdav/common/datastore/sql_schema/old/postgres-dialect/v15.sql
    CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_15_to_16.sql
    CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_15_to_16.sql

Removed Paths:
-------------
    CalendarServer/trunk/twistedcaldav/notify.py
    CalendarServer/trunk/twistedcaldav/test/test_notify.py

Property Changed:
----------------
    CalendarServer/trunk/


Property changes on: CalendarServer/trunk
___________________________________________________________________
Modified: svn:ignore
   - .dependencies
*.tgz
data
logs
build
*.pyc
*.pyo
_run
.settings
.project
.pydevproject

   + .dependencies
*.tgz
data
logs
build
*.pyc
*.pyo
_run
.settings
.project
.pydevproject
parsecalendardata.py
dopstats.py


Modified: CalendarServer/trunk/calendarserver/provision/root.py
===================================================================
--- CalendarServer/trunk/calendarserver/provision/root.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/provision/root.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -208,31 +208,6 @@
         for filter in self.contentFilters:
             request.addResponseFilter(filter[0], atEnd=filter[1])
 
-        # Examine headers for our special internal authorization, used for
-        # POSTing to /inbox between workers and mail gateway sidecar.
-        if not hasattr(request, "checkedInternalAuthHeader"):
-            request.checkedInternalAuthHeader = True
-            headerName = config.Scheduling.iMIP.Header
-            secrets = request.headers.getRawHeaders(headerName, None)
-            secretVerified = False
-            if secrets is not None:
-                log.debug("Internal authentication header (%s) detected" %
-                    (headerName,))
-                for secret in secrets:
-                    if secret == config.Scheduling.iMIP.Password:
-                        secretVerified = True
-                        break
-
-            if secretVerified:
-                log.debug("Internal authentication header (%s) verified" %
-                    (headerName,))
-                guid = config.Scheduling.iMIP.GUID
-                log.debug("Internal principal %s being assigned to authnUser and authzUser" % (guid,))
-                request.authzUser = request.authnUser = davxml.Principal(
-                    davxml.HRef.fromString("/principals/__uids__/%s/" % (guid,))
-                )
-
-
         # Examine cookies for wiki auth token; if there, ask the paired wiki
         # server for the corresponding record name.  If that maps to a
         # principal, assign that to authnuser.

Modified: CalendarServer/trunk/calendarserver/provision/test/test_root.py
===================================================================
--- CalendarServer/trunk/calendarserver/provision/test/test_root.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/provision/test/test_root.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -36,7 +36,6 @@
 
 from calendarserver.provision.root import RootResource
 from twistedcaldav.directory import augment
-from twistedcaldav.config import config
 
 class FakeCheckSACL(object):
     def __init__(self, sacls=None):
@@ -298,35 +297,8 @@
         except HTTPError, e:
             self.assertEquals(e.response.code, 401)
 
-    @inlineCallbacks
-    def test_internalAuthHeader(self):
-        """
-        Test the behavior of locateChild when x-calendarserver-internal
-        header is set.
 
-        authnuser and authzuser will be set to the internal principal
-        """
-        self.patch(config.Scheduling.iMIP, "Password", "xyzzy")
 
-        headers = http_headers.Headers({})
-        headers.setRawHeaders("x-calendarserver-internal", ["xyzzy"])
-
-        request = SimpleRequest(
-            self.site,
-            "GET",
-            "/principals/",
-            headers=headers,
-        )
-
-        resrc, segments = (yield
-            RootResource.locateChild(self.root.resource, request, ["principals"]
-        ))
-
-        expected = "<?xml version='1.0' encoding='UTF-8'?>\n<principal xmlns='DAV:'>\r\n  <href>/principals/__uids__/%s/</href>\r\n</principal>" % (config.Scheduling.iMIP.GUID,)
-        self.assertEquals(request.authnUser.toxml(), expected)
-        self.assertEquals(request.authzUser.toxml(), expected)
-
-
     def test_DELETE(self):
         def do_test(response):
             response = IResponse(response)

Modified: CalendarServer/trunk/calendarserver/push/amppush.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/amppush.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/push/amppush.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -21,7 +21,6 @@
 from twisted.internet.endpoints import TCP4ClientEndpoint, TCP4ServerEndpoint
 from twisted.internet.protocol import Factory, ServerFactory
 from twisted.protocols import amp
-from twistedcaldav.notify import getPubSubPath
 import time
 import uuid
 
@@ -50,6 +49,7 @@
 
 # Server classes
 
+
 class AMPPushNotifierService(StreamServerEndpointService, LoggingMixIn):
     """
     AMPPushNotifierService allows clients to use AMP to subscribe to,
@@ -57,16 +57,17 @@
     """
 
     @classmethod
-    def makeService(cls, settings, ignored, serverHostName, reactor=None):
-        return cls(settings, serverHostName, reactor=reactor)
+    def makeService(cls, settings, ignored, reactor=None):
+        return cls(settings, reactor=reactor)
 
-    def __init__(self, settings, serverHostName, reactor=None):
+    def __init__(self, settings, reactor=None):
         if reactor is None:
             from twisted.internet import reactor
         factory = AMPPushNotifierFactory(self)
         endpoint = TCP4ServerEndpoint(reactor, settings["Port"])
         super(AMPPushNotifierService, self).__init__(endpoint, factory)
         self.subscribers = []
+        self.dataHost = settings["DataHost"]
 
         if settings["EnableStaggering"]:
             self.scheduler = PushScheduler(reactor, self.sendNotification,
@@ -74,8 +75,6 @@
         else:
             self.scheduler = None
 
-        self.serverHostName = serverHostName
-
     def addSubscriber(self, p):
         self.log_debug("Added subscriber")
         self.subscribers.append(p)
@@ -84,13 +83,10 @@
         self.log_debug("Removed subscriber")
         self.subscribers.remove(p)
 
-    def enqueue(self, op, id, dataChangedTimestamp=None):
+    def enqueue(self, id, dataChangedTimestamp=None):
         """
         Sends an AMP push notification to any clients subscribing to this id.
 
-        @param op: The operation that took place, either "create" or "update"
-            (ignored in this implementation)
-        @type op: C{str}
         @param id: The identifier of the resource that was updated, including
             a prefix indicating whether this is CalDAV or CardDAV related.
             The prefix is separated from the id with "|", e.g.:
@@ -107,18 +103,18 @@
         """
 
         try:
-            id.split("|", 1)
+            protocol, id = id.split("|", 1)
         except ValueError:
             # id has no protocol, so we can't do anything with it
             self.log_error("Notification id '%s' is missing protocol" % (id,))
             return
 
-        id = getPubSubPath(id, {"host": self.serverHostName})
-
         # Unit tests can pass this value in; otherwise it defaults to now
         if dataChangedTimestamp is None:
             dataChangedTimestamp = int(time.time())
 
+        id = "/%s/%s/%s/" % (protocol, self.dataHost, id)
+
         tokens = []
         for subscriber in self.subscribers:
             token = subscriber.subscribedToID(id)

Modified: CalendarServer/trunk/calendarserver/push/applepush.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/applepush.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/push/applepush.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -59,7 +59,7 @@
     """
 
     @classmethod
-    def makeService(cls, settings, store, serverHostName, testConnectorClass=None,
+    def makeService(cls, settings, store, testConnectorClass=None,
         reactor=None):
         """
         Creates the various "subservices" that work together to implement
@@ -177,14 +177,11 @@
 
 
     @inlineCallbacks
-    def enqueue(self, op, id, dataChangedTimestamp=None):
+    def enqueue(self, id, dataChangedTimestamp=None):
         """
         Sends an Apple Push Notification to any device token subscribed to
         this id.
 
-        @param op: The operation that took place, either "create" or "update"
-            (ignored in this implementation)
-        @type op: C{str}
         @param id: The identifier of the resource that was updated, including
             a prefix indicating whether this is CalDAV or CardDAV related.
             The prefix is separated from the id with "|", e.g.:

Added: CalendarServer/trunk/calendarserver/push/notifier.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/notifier.py	                        (rev 0)
+++ CalendarServer/trunk/calendarserver/push/notifier.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,209 @@
+##
+# Copyright (c) 2005-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Notification framework for Calendar Server
+"""
+
+from twext.python.log import LoggingMixIn, Logger
+
+from twisted.internet.defer import inlineCallbacks, succeed
+from twext.enterprise.dal.record import fromTable
+from twext.enterprise.queue import WorkItem
+from txdav.common.datastore.sql_tables import schema
+from twisted.application import service
+from twisted.python.reflect import namedClass
+
+
+log = Logger()
+
+
+class PushNotificationWork(WorkItem, fromTable(schema.PUSH_NOTIFICATION_WORK)):
+
+    @inlineCallbacks
+    def doWork(self):
+
+        # FIXME: Coalescing goes here?
+
+        pushService = self.transaction._pushService
+        if pushService is not None:
+            yield pushService.enqueue(self.pushID)
+
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Classes used within calendarserver itself
+#
+
+class Notifier(LoggingMixIn):
+    """
+    Provides a hook for sending change notifications to the
+    L{NotifierFactory}.
+    """
+
+    def __init__(self, notifierFactory, label="default", id=None, prefix=None):
+        self._notifierFactory = notifierFactory
+        self._ids = { label : self.normalizeID(id) }
+        self._notify = True
+        self._prefix = prefix
+
+    def normalizeID(self, id):
+        urn = "urn:uuid:"
+        try:
+            if id.startswith(urn):
+                return id[len(urn):]
+        except AttributeError:
+            pass
+        return id
+
+    def enableNotify(self, arg):
+        self.log_debug("enableNotify: %s" % (self._ids['default'][1],))
+        self._notify = True
+
+    def disableNotify(self):
+        self.log_debug("disableNotify: %s" % (self._ids['default'][1],))
+        self._notify = False
+
+    @inlineCallbacks
+    def notify(self):
+        for label in self._ids.iterkeys():
+            id = self.getID(label=label)
+            if id is not None:
+                if self._notify:
+                    self.log_debug("Notifications are enabled: %s %s" %
+                        (label, id))
+                    yield self._notifierFactory.send(id)
+                else:
+                    self.log_debug("Skipping notification for: %s" % (id,))
+
+    def clone(self, label="default", id=None):
+        newNotifier = self.__class__(self._notifierFactory)
+        newNotifier._ids = self._ids.copy()
+        newNotifier._ids[label] = id
+        newNotifier._prefix = self._prefix
+        return newNotifier
+
+    def addID(self, label="default", id=None):
+        self._ids[label] = self.normalizeID(id)
+
+    def getID(self, label="default"):
+        id = self._ids.get(label, None)
+        if self._prefix is None:
+            return id
+        else:
+            return "%s|%s" % (self._prefix, id)
+
+    def nodeName(self, label="default"):
+        id = self.getID(label=label)
+        return succeed(self._notifierFactory.pushKeyForId(id))
+
+
+class NotifierFactory(LoggingMixIn):
+    """
+    Notifier Factory
+
+    Creates Notifier instances and forwards notifications from them to the
+    work queue.
+    """
+
+    def __init__(self, store, hostname, reactor=None):
+        self.store = store
+        self.hostname = hostname
+
+        if reactor is None:
+            from twisted.internet import reactor
+        self.reactor = reactor
+
+    @inlineCallbacks
+    def send(self, id):
+        txn = self.store.newTransaction()
+        yield txn.enqueue(PushNotificationWork, pushID=id)
+        yield txn.commit()
+
+    def newNotifier(self, label="default", id=None, prefix=None):
+        return Notifier(self, label=label, id=id, prefix=prefix)
+
+    def pushKeyForId(self, id):
+        path = "/"
+
+        try:
+            prefix, id = id.split("|", 1)
+            path += "%s/" % (prefix,)
+        except ValueError:
+            # id has no prefix
+            pass
+
+        path += "%s/" % (self.hostname,)
+        if id:
+            path += "%s/" % (id,)
+        return path
+
+
+
+def getPubSubAPSConfiguration(id, config):
+    """
+    Returns the Apple push notification settings specific to the notifier
+    ID, which includes a prefix that is either "CalDAV" or "CardDAV"
+    """
+    try:
+        prefix, id = id.split("|", 1)
+    except ValueError:
+        # id has no prefix, so we can't look up APS config
+        return None
+
+    # If we are directly talking to apple push, advertise those settings
+    applePushSettings = config.Notifications.Services.ApplePushNotifier
+    if applePushSettings.Enabled:
+        settings = {}
+        settings["APSBundleID"] = applePushSettings[prefix]["Topic"]
+        if config.EnableSSL:
+            url = "https://%s:%s/%s" % (config.ServerHostName, config.SSLPort,
+                applePushSettings.SubscriptionURL)
+        else:
+            url = "http://%s:%s/%s" % (config.ServerHostName, config.HTTPPort,
+                applePushSettings.SubscriptionURL)
+        settings["SubscriptionURL"] = url
+        settings["SubscriptionRefreshIntervalSeconds"] = applePushSettings.SubscriptionRefreshIntervalSeconds
+        settings["APSEnvironment"] = applePushSettings.Environment
+        return settings
+
+    return None
+
+
+class PushService(service.MultiService):
+    """
+    A Service which passes along notifications to the protocol-specific subservices
+    """
+
+    @classmethod
+    def makeService(cls, settings, store):
+        multiService = cls()
+        for key, subSettings in settings.Services.iteritems():
+            if subSettings["Enabled"]:
+                subService = namedClass(subSettings["Service"]).makeService(
+                    subSettings, store)
+                subService.setServiceParent(multiService)
+                multiService.subServices.append(subService)            
+        return multiService
+
+    def __init__(self):
+        service.MultiService.__init__(self)
+        self.subServices = []
+
+    @inlineCallbacks
+    def enqueue(self, id):
+        for subService in self.subServices:
+            yield subService.enqueue(id)

Modified: CalendarServer/trunk/calendarserver/push/test/test_amppush.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/test/test_amppush.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/push/test/test_amppush.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -31,12 +31,13 @@
             "Port" : 62311,
             "EnableStaggering" : True,
             "StaggerSeconds" : 3,
+            "DataHost" : "localhost",
         }
 
         # Set up the service
         clock = Clock()
         service = (yield AMPPushNotifierService.makeService(settings,
-            None, "localhost", reactor=clock))
+            None, reactor=clock))
 
         self.assertEquals(service.subscribers, [])
 
@@ -68,7 +69,7 @@
         self.assertTrue(client3.subscribedToID("/CalDAV/localhost/user03/"))
 
         dataChangedTimestamp = 1354815999
-        service.enqueue("update", "CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
+        service.enqueue("CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
         self.assertEquals(len(client1.history), 0)
         self.assertEquals(len(client2.history), 0)
         self.assertEquals(len(client3.history), 0)
@@ -85,7 +86,7 @@
         client1.reset()
         client2.reset()
         client2.unsubscribe("token2", "/CalDAV/localhost/user01/")
-        service.enqueue("update", "CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
+        service.enqueue("CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
         self.assertEquals(len(client1.history), 0)
         clock.advance(1)
         self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp' : 1354815999})])
@@ -98,7 +99,7 @@
         client1.reset()
         client2.reset()
         client2.subscribe("token2", "/CalDAV/localhost/user01/")
-        service.enqueue("update", "CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
+        service.enqueue("CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
         self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp' : 1354815999})])
         self.assertEquals(client2.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp' : 1354815999})])
 

Modified: CalendarServer/trunk/calendarserver/push/test/test_applepush.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/test/test_applepush.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/push/test/test_applepush.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -118,7 +118,7 @@
         # Set up the service
         clock = Clock()
         service = (yield ApplePushNotifierService.makeService(settings,
-            self.store, "localhost", testConnectorClass=TestConnector, reactor=clock))
+            self.store, testConnectorClass=TestConnector, reactor=clock))
         self.assertEquals(set(service.providers.keys()), set(["CalDAV","CardDAV"]))
         self.assertEquals(set(service.feedbacks.keys()), set(["CalDAV","CardDAV"]))
 
@@ -127,7 +127,7 @@
 
         # Notification arrives from calendar server
         dataChangedTimestamp = 1354815999
-        yield service.enqueue("update", "CalDAV|user01/calendar",
+        yield service.enqueue("CalDAV|user01/calendar",
             dataChangedTimestamp=dataChangedTimestamp)
 
         # The notifications should be in the queue
@@ -167,7 +167,7 @@
         # Reset sent data
         providerConnector.transport.data = None
         # Send notification while service is connected
-        yield service.enqueue("update", "CalDAV|user01/calendar")
+        yield service.enqueue("CalDAV|user01/calendar")
         clock.advance(1) # so that first push is sent
         self.assertEquals(len(providerConnector.transport.data), 183)
         # Reset sent data

Added: CalendarServer/trunk/calendarserver/push/test/test_notifier.py
===================================================================
--- CalendarServer/trunk/calendarserver/push/test/test_notifier.py	                        (rev 0)
+++ CalendarServer/trunk/calendarserver/push/test/test_notifier.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,57 @@
+##
+# Copyright (c) 2011-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twistedcaldav.test.util import TestCase
+from twistedcaldav.config import ConfigDict
+from calendarserver.push.notifier import PushService
+from twisted.internet.defer import inlineCallbacks, succeed
+from twisted.application import service
+
+class StubService(service.Service):
+    def __init__(self, settings, store):
+        self.settings = settings
+        self.store = store
+        self.reset()
+
+    def reset(self):
+        self.history = []
+
+    def enqueue(self, id):
+        self.history.append(id)
+        return(succeed(None))
+
+    @classmethod
+    def makeService(cls, settings, store):
+        return cls(settings, store)
+
+class PushServiceTests(TestCase):
+
+    @inlineCallbacks
+    def test_enqueue(self):
+        settings = ConfigDict({
+            "Services" : {
+                "Stub" : {
+                    "Service" : "calendarserver.push.test.test_notifier.StubService",
+                    "Enabled" : True,
+                    "Foo" : "Bar",
+                },
+            },
+        })
+        svc = PushService.makeService(settings, None)
+        yield svc.enqueue("testing")
+        self.assertEquals(svc.subServices[0].history, ["testing"])
+
+

Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -70,7 +70,6 @@
 from twistedcaldav.config import ConfigurationError
 from twistedcaldav.config import config
 from twistedcaldav.localization import processLocalizationFiles
-from twistedcaldav.scheduling.imip.resource import IMIPReplyInboxResource
 from twistedcaldav import memcachepool
 from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
 from twistedcaldav.upgrade import UpgradeFileSystemFormatService, PostDBImportService
@@ -106,6 +105,8 @@
 from calendarserver.tap.util import oracleConnectorFromConfig
 from calendarserver.tap.cfgchild import ConfiguredChildSpawner
 from calendarserver.tools.util import checkDirectory
+from calendarserver.push.notifier import PushService
+from twistedcaldav.scheduling.imip.inbound import MailRetriever
 
 try:
     from calendarserver.version import version
@@ -510,8 +511,6 @@
     L{DelayedStartupProcessMonitor}:
 
         - regular slave processes (CalDAV workers)
-        - notifier
-        - mail gateway
     """
 
     def __init__(self, maker, monitor, dispenser, dispatcher, configPath,
@@ -540,51 +539,7 @@
             )
             self.monitor.addProcessObject(process, PARENT_ENVIRONMENT)
 
-        if (
-            config.Notifications.Enabled and
-            config.Notifications.InternalNotificationHost == "localhost"
-        ):
-            self.maker.log_info("Adding notification service")
 
-            notificationsArgv = [
-                sys.executable,
-                sys.argv[0],
-            ]
-            if config.UserName:
-                notificationsArgv.extend(("-u", config.UserName))
-            if config.GroupName:
-                notificationsArgv.extend(("-g", config.GroupName))
-            notificationsArgv.extend((
-                "--reactor=%s" % (config.Twisted.reactor,),
-                "-n", self.maker.notifierTapName,
-                "-f", self.configPath,
-            ))
-            self.monitor.addProcess("notifications", notificationsArgv,
-                env=PARENT_ENVIRONMENT)
-
-        if (
-            config.Scheduling.iMIP.Enabled and
-            config.Scheduling.iMIP.MailGatewayServer == "localhost"
-        ):
-            self.maker.log_info("Adding mail gateway service")
-
-            mailGatewayArgv = [
-                sys.executable,
-                sys.argv[0],
-            ]
-            if config.UserName:
-                mailGatewayArgv.extend(("-u", config.UserName))
-            if config.GroupName:
-                mailGatewayArgv.extend(("-g", config.GroupName))
-            mailGatewayArgv.extend((
-                "--reactor=%s" % (config.Twisted.reactor,),
-                "-n", self.maker.mailGatewayTapName,
-                "-f", self.configPath,
-            ))
-
-            self.monitor.addProcess("mailgateway", mailGatewayArgv,
-                               env=PARENT_ENVIRONMENT)
-
         if config.GroupCaching.Enabled and config.GroupCaching.EnableUpdater:
             self.maker.log_info("Adding group caching service")
 
@@ -667,8 +622,6 @@
     #
     # Default tap names
     #
-    mailGatewayTapName = "caldav_mailgateway"
-    notifierTapName = "caldav_notifier"
     groupMembershipCacherTapName = "caldav_groupcacher"
 
 
@@ -758,9 +711,33 @@
         pool, txnFactory = getDBPool(config)
         store = storeFromConfig(config, txnFactory)
         result = self.requestProcessingService(options, store)
+        directory = result.rootResource.getDirectory()
         if pool is not None:
             pool.setServiceParent(result)
 
+
+        # Optionally set up push notifications
+        if config.Notifications.Enabled:
+            pushService = PushService.makeService(config.Notifications, store)
+            pushService.setServiceParent(result)
+        else:
+            pushService = None
+
+        # Optionally set up mail retrieval
+        if config.Scheduling.iMIP.Enabled:
+            mailRetriever = MailRetriever(store, directory,
+                config.Scheduling.iMIP.Receiving)
+            mailRetriever.setServiceParent(result)
+        else:
+            mailRetriever = None
+
+        def decorateTransaction(txn):
+            txn._pushService = pushService
+            txn._rootResource = result.rootResource
+            txn._mailRetriever = mailRetriever
+
+        store.callWithNewTransactions(decorateTransaction)
+
         # Optionally enable Manhole access
         if config.Manhole.Enabled:
             try:
@@ -773,7 +750,7 @@
                         "config" : config,
                         "service" : result,
                         "store" : store,
-                        "directory" : result.rootResource.getDirectory(),
+                        "directory" : directory,
                         },
                     "passwd" : config.Manhole.PasswordFilePath,
                 })
@@ -801,9 +778,9 @@
         oldLogLevel = logLevelForNamespace(None)
         setLogLevelForNamespace(None, "info")
 
+        # Note: 'additional' was used for IMIP reply resource, and perhaps
+        # we can remove this
         additional = []
-        if config.Scheduling.iMIP.Enabled:
-            additional.append(("inbox", IMIPReplyInboxResource, [], ("digest",)))
 
         #
         # Configure the service
@@ -836,7 +813,7 @@
             from txdav.common.datastore.sql import CommonDataStore as SQLStore
             if isinstance(store, SQLStore):
                 def queueMasterAvailable(connectionFromMaster):
-                    store.queuer = connectionFromMaster
+                    store.queuer = store.queuer.transferProposalCallbacks(connectionFromMaster)
                 queueFactory = QueueWorkerFactory(store.newTransaction, schema,
                                                   queueMasterAvailable)
                 controlSocketClient.addFactory(_QUEUE_ROUTE, queueFactory)
@@ -1352,6 +1329,7 @@
             except ImportError:
                 print "Manhole access could not enabled because manhole_tap could not be imported"
 
+
         # Finally, let's get the real show on the road.  Create a service that
         # will spawn all of our worker processes when started, and wrap that
         # service in zero to two necessary layers before it's started: first,

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -57,7 +57,7 @@
 from twistedcaldav.directory.internal import InternalDirectoryService
 from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
 from twistedcaldav.directory.wiki import WikiDirectoryService
-from twistedcaldav.notify import NotifierFactory, getPubSubConfiguration
+from calendarserver.push.notifier import NotifierFactory 
 from calendarserver.push.applepush import APNSubscriptionResource
 from twistedcaldav.directorybackedaddressbook import DirectoryBackedAddressBookResource
 from twistedcaldav.resource import AuthenticationWrapper
@@ -223,11 +223,9 @@
     # Configure NotifierFactory
     #
     if config.Notifications.Enabled:
-        notifierFactory = NotifierFactory(
-            config.Notifications.InternalNotificationHost,
-            config.Notifications.InternalNotificationPort,
-            pubSubConfig=getPubSubConfiguration(config)
-        )
+        # FIXME: NotifierFactory needs reference to the store in order
+        # to get a txn in order to create a Work item
+        notifierFactory = NotifierFactory(None, config.ServerHostName)
     else:
         notifierFactory = None
     quota = config.UserQuota
@@ -239,7 +237,7 @@
         else:
             uri = "http://%s:%s" % (config.ServerHostName, config.HTTPPort,)
         attachments_uri = uri + "/calendars/__uids__/%(home)s/dropbox/%(dropbox_id)s/%(name)s"
-        return CommonSQLDataStore(
+        store = CommonSQLDataStore(
             txnFactory, notifierFactory,
             FilePath(config.AttachmentsRoot), attachments_uri,
             config.EnableCalDAV, config.EnableCardDAV,
@@ -256,11 +254,14 @@
             cacheExpireSeconds=config.QueryCaching.ExpireSeconds
         )
     else:
-        return CommonFileDataStore(
+        store = CommonFileDataStore(
             FilePath(config.DocumentRoot),
             notifierFactory, config.EnableCalDAV, config.EnableCardDAV,
             quota=quota
         )
+    if notifierFactory is not None:
+        notifierFactory.store = store
+    return store
 
 
 
@@ -268,7 +269,6 @@
     """
     Create an L{AggregateDirectoryService} from the given configuration.
     """
-
     #
     # Setup the Augment Service
     #
@@ -362,7 +362,7 @@
 
 
 
-def getRootResource(config, newStore, resources=None):
+def getRootResource(config, newStore, resources=None, directory=None):
     """
     Set up directory service and resource hierarchy based on config.
     Return root resource.
@@ -398,7 +398,8 @@
     directoryBackedAddressBookResourceClass = DirectoryBackedAddressBookResource
     apnSubscriptionResourceClass = APNSubscriptionResource
 
-    directory = directoryFromConfig(config)
+    if directory is None:
+        directory = directoryFromConfig(config)
 
     #
     # Setup the ProxyDB Service
@@ -706,6 +707,10 @@
         directory,
     )
 
+    # FIXME:  Storing a reference to the root resource on the store
+    # until scheduling no longer needs resource objects
+    newStore.rootResource = root
+
     return logWrapper
 
 
@@ -788,7 +793,7 @@
 
 class FakeRequest(object):
 
-    def __init__(self, rootResource, method, path, uri='/'):
+    def __init__(self, rootResource, method, path, uri='/', transaction=None):
         self.rootResource = rootResource
         self.method = method
         self.path = path
@@ -796,6 +801,8 @@
         self._resourcesByURL = {}
         self._urlsByResource = {}
         self.headers = Headers()
+        if transaction is not None:
+            self._newStoreTransaction = transaction
 
 
     @inlineCallbacks

Modified: CalendarServer/trunk/calendarserver/tools/changeip_calendar.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/changeip_calendar.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/tools/changeip_calendar.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -98,9 +98,6 @@
     keys = (
         ("Authentication", "Wiki", "Hostname"),
         ("BindAddresses",),
-        ("Notifications", "Services", "XMPPNotifier", "Host"),
-        ("Notifications", "Services", "XMPPNotifier", "JID"),
-        ("Notifications", "Services", "XMPPNotifier", "ServiceAddress"),
         ("Scheduling", "iMIP", "Receiving", "Server"),
         ("Scheduling", "iMIP", "Sending", "Server"),
         ("Scheduling", "iMIP", "Sending", "Address"),

Modified: CalendarServer/trunk/calendarserver/tools/test/test_changeip.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_changeip.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/tools/test/test_changeip.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -35,14 +35,6 @@
                 "original_hostname",
             ],
             "ServerHostName" : "",
-            "Notifications" : {
-                "Services" : {
-                    "XMPPNotifier" : {
-                        "Host" : "original_hostname",
-                        "JID" : "com.apple.notificationuser at original_hostname",
-                    },
-                },
-            },
             "Scheduling" : {
                 "iMIP" : {
                     "Receiving" : {
@@ -74,14 +66,6 @@
                     "new_hostname",
                 ],
                 "ServerHostName" : "",
-                "Notifications" : {
-                    "Services" : {
-                        "XMPPNotifier" : {
-                            "Host" : "new_hostname",
-                            "JID" : "com.apple.notificationuser at new_hostname",
-                        },
-                    },
-                },
                 "Scheduling" : {
                     "iMIP" : {
                         "Receiving" : {

Modified: CalendarServer/trunk/calendarserver/tools/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/util.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/calendarserver/tools/util.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -44,7 +44,7 @@
 from twistedcaldav.directory import calendaruserproxy
 from twistedcaldav.directory.aggregate import AggregateDirectoryService
 from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
-from twistedcaldav.notify import NotifierFactory
+from calendarserver.push.notifier import NotifierFactory
 from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
 
 from txdav.common.datastore.file import CommonDataStore
@@ -77,9 +77,10 @@
             if not hasattr(self, "_principalCollection"):
 
                 if config.Notifications.Enabled:
+                    # FIXME: NotifierFactory needs reference to the store in order
+                    # to get a txn in order to create a Work item
                     notifierFactory = NotifierFactory(
-                        config.Notifications.InternalNotificationHost,
-                        config.Notifications.InternalNotificationPort,
+                        None, config.ServerHostName,
                     )
                 else:
                     notifierFactory = None
@@ -87,6 +88,8 @@
                 # Need a data store
                 _newStore = CommonDataStore(FilePath(config.DocumentRoot), 
                     notifierFactory, True, False)
+                if notifierFactory is not None:
+                    notifierFactory.store = _newStore
 
                 #
                 # Instantiating a DirectoryCalendarHomeProvisioningResource with a directory

Modified: CalendarServer/trunk/contrib/performance/benchmarks/find_calendars.py
===================================================================
--- CalendarServer/trunk/contrib/performance/benchmarks/find_calendars.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/contrib/performance/benchmarks/find_calendars.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -32,8 +32,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <x0:propfind xmlns:x0="DAV:" xmlns:x3="http://apple.com/ns/ical/" xmlns:x1="http://calendarserver.org/ns/" xmlns:x2="urn:ietf:params:xml:ns:caldav">
  <x0:prop>
-  <x1:xmpp-server/>
-  <x1:xmpp-uri/>
   <x1:getctag/>
   <x0:displayname/>
   <x2:calendar-description/>

Modified: CalendarServer/trunk/twext/enterprise/ienterprise.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/ienterprise.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twext/enterprise/ienterprise.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -296,3 +296,17 @@
         @rtype: L{twext.enterprise.queue.WorkItem}
         """
 
+
+    def callWithNewProposals(self, callback):
+        """
+        Tells the IQueuer to call a callback method whenever a new WorkProposal
+        is created.
+
+        @param callback: a callable which accepts a single parameter, a
+            L{WorkProposal}
+        """
+
+    def transferProposalCallbacks(self, newQueuer):
+        """
+        Transfer the registered callbacks to the new queuer.
+        """
\ No newline at end of file

Modified: CalendarServer/trunk/twext/enterprise/queue.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/queue.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twext/enterprise/queue.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -710,6 +710,7 @@
         return result
 
 
+    @PerformWork.responder
     def performWork(self, table, workID):
         """
         Dispatch work to this worker.
@@ -742,6 +743,9 @@
                                                        boxReceiver, locator)
         self.transactionFactory = transactionFactory
         self.whenConnected = whenConnected
+        # FIXME: Glyph it appears WorkProposal expects this to have reactor...
+        from twisted.internet import reactor
+        self.reactor = reactor
 
 
     def startReceivingBoxes(self, sender):
@@ -1012,9 +1016,45 @@
         """
         return _cloneDeferred(self._whenCommitted)
 
+class _BaseQueuer(object):
+    implements(IQueuer)
 
+    def __init__(self):
+        super(_BaseQueuer, self).__init__()
+        self.proposalCallbacks = set()
 
-class PeerConnectionPool(MultiService, object):
+    def callWithNewProposals(self, callback):
+        self.proposalCallbacks.add(callback);
+
+    def transferProposalCallbacks(self, newQueuer):
+        newQueuer.proposalCallbacks = self.proposalCallbacks
+        return newQueuer
+
+    def enqueueWork(self, txn, workItemType, **kw):
+        """
+        There is some work to do.  Do it, someplace else, ideally in parallel.
+        Later, let the caller know that the work has been completed by firing a
+        L{Deferred}.
+
+        @param workItemType: The type of work item to be enqueued.
+        @type workItemType: A subtype of L{WorkItem}
+
+        @param kw: The parameters to construct a work item.
+        @type kw: keyword parameters to C{workItemType.create}, i.e.
+            C{workItemType.__init__}
+
+        @return: an object that can track the enqueuing and remote execution of
+            this work.
+        @rtype: L{WorkProposal}
+        """
+        wp = WorkProposal(self, txn, workItemType, kw)
+        wp._start()
+        for callback in self.proposalCallbacks:
+            callback(wp)
+        return wp
+
+
+class PeerConnectionPool(_BaseQueuer, MultiService, object):
     """
     Each node has a L{PeerConnectionPool} connecting it to all the other nodes
     currently active on the same database.
@@ -1153,28 +1193,8 @@
         return self.choosePerformer(onlyLocally=True).performWork(table, workID)
 
 
-    def enqueueWork(self, txn, workItemType, **kw):
-        """
-        There is some work to do.  Do it, someplace else, ideally in parallel.
-        Later, let the caller know that the work has been completed by firing a
-        L{Deferred}.
 
-        @param workItemType: The type of work item to be enqueued.
-        @type workItemType: A subtype of L{WorkItem}
 
-        @param kw: The parameters to construct a work item.
-        @type kw: keyword parameters to C{workItemType.create}, i.e.
-            C{workItemType.__init__}
-
-        @return: an object that can track the enqueuing and remote execution of
-            this work.
-        @rtype: L{WorkProposal}
-        """
-        wp = WorkProposal(self, txn, workItemType, kw)
-        wp._start()
-        return wp
-
-
     def allWorkItemTypes(self):
         """
         Load all the L{WorkItem} types that this node can process and return
@@ -1397,13 +1417,16 @@
 
 
 
-class LocalQueuer(object):
+
+
+class LocalQueuer(_BaseQueuer):
     """
     When work is enqueued with this queuer, it is just executed locally.
     """
     implements(IQueuer)
 
     def __init__(self, txnFactory, reactor=None):
+        super(LocalQueuer, self).__init__()
         self.txnFactory = txnFactory
         if reactor is None:
             from twisted.internet import reactor
@@ -1414,23 +1437,4 @@
         """
         Choose to perform the work locally.
         """
-        return LocalPerformer(self.txnFactory)
-
-
-    def enqueueWork(self, txn, workItemType, **kw):
-        """
-        Do this work in the local process.
-
-        @see: L{PeerConnectionPool.enqueueWork}
-
-        @return: a pseudo work proposal, since everything completes at the same
-            time.
-        @rtype: L{WorkProposal}
-        """
-        wp = WorkProposal(self, txn, workItemType, kw)
-        wp._start()
-        return wp
-
-
-
-
+        return LocalPerformer(self.txnFactory)
\ No newline at end of file

Modified: CalendarServer/trunk/twext/enterprise/test/test_queue.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/test/test_queue.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twext/enterprise/test/test_queue.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -54,6 +54,9 @@
 from zope.interface.verify import verifyObject
 from twisted.test.proto_helpers import StringTransport
 
+from twext.enterprise.queue import _BaseQueuer
+import twext.enterprise.queue
+
 class Clock(_Clock):
     """
     More careful L{IReactorTime} fake which mimics the exception behavior of
@@ -334,7 +337,7 @@
 
 
     @inlineCallbacks
-    def test_notBeforeWhenCheckingForLostWork(self):
+    def FIXME_test_notBeforeWhenCheckingForLostWork(self):
         """
         L{PeerConnectionPool._periodicLostWorkCheck} should execute any
         outstanding work items, but only those that are expired.
@@ -627,3 +630,27 @@
         self.assertEquals(rows, [[4321, 7]])
 
 
+class DummyProposal(object):
+
+    def __init__(self, *ignored):
+        pass
+
+    def _start(self):
+        pass
+
+class BaseQueuerTests(TestCase):
+
+    def setUp(self):
+        self.proposal = None
+        self.patch(twext.enterprise.queue, "WorkProposal", DummyProposal)
+
+    def _proposalCallback(self, proposal):
+        self.proposal = proposal
+
+    def test_proposalCallbacks(self):
+        queuer = _BaseQueuer()
+        queuer.callWithNewProposals(self._proposalCallback)
+        self.assertEqual(self.proposal, None)
+        queuer.enqueueWork(None, None)
+        self.assertNotEqual(self.proposal, None)
+

Modified: CalendarServer/trunk/twisted/plugins/caldav.py
===================================================================
--- CalendarServer/trunk/twisted/plugins/caldav.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twisted/plugins/caldav.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -53,6 +53,4 @@
 
 
 TwistedCalDAV = TAP("calendarserver.tap.caldav.CalDAVServiceMaker")
-CalDAVNotifier = TAP("twistedcaldav.notify.NotificationServiceMaker")
-CalDAVMailGateway = TAP("twistedcaldav.scheduling.imip.mailgateway.MailGatewayServiceMaker")
 CalDAVGroupCacher = TAP("twistedcaldav.directory.directory.GroupMembershipCacherServiceMaker")

Modified: CalendarServer/trunk/twistedcaldav/customxml.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/customxml.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/customxml.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -418,8 +418,6 @@
         (calendarserver_namespace, "subscription-url") : (1, 1),
         (calendarserver_namespace, "apsbundleid") : (1, 1),
         (calendarserver_namespace, "env") : (1, 1),
-        (calendarserver_namespace, "xmpp-server") : (1, 1),
-        (calendarserver_namespace, "xmpp-uri") : (1, 1),
     }
 
 
@@ -470,66 +468,8 @@
 
 
 
- at registerElement
-class PubSubXMPPURIProperty (WebDAVTextElement):
-    """
-    A calendar home property to indicate the pubsub XMPP URI to subscribe to
-    for notifications.
-    """
-    namespace = calendarserver_namespace
-    name = "xmpp-uri"
-    protected = True
-    hidden = True
 
 
-
- at registerElement
-class PubSubHeartbeatProperty (WebDAVElement):
-    """
-    A calendar home property to indicate the pubsub XMPP URI to subscribe to
-    for server heartbeats.
-    """
-    namespace = calendarserver_namespace
-    name = "xmpp-heartbeat"
-    protected = True
-    hidden = True
-    allowed_children = {
-        (calendarserver_namespace, "xmpp-heartbeat-uri")     : (1, 1),
-        (calendarserver_namespace, "xmpp-heartbeat-minutes") : (1, 1),
-    }
-
-
-
- at registerElement
-class PubSubHeartbeatURIProperty (WebDAVTextElement):
-    namespace = calendarserver_namespace
-    name = "xmpp-heartbeat-uri"
-    protected = True
-    hidden = True
-
-
-
- at registerElement
-class PubSubHeartbeatMinutesProperty (WebDAVTextElement):
-    namespace = calendarserver_namespace
-    name = "xmpp-heartbeat-minutes"
-    protected = True
-    hidden = True
-
-
-
- at registerElement
-class PubSubXMPPServerProperty (WebDAVTextElement):
-    """
-    A calendar home property to indicate the pubsub XMPP hostname to
-    contact for notifications.
-    """
-    namespace = calendarserver_namespace
-    name = "xmpp-server"
-    protected = True
-    hidden = True
-
-
 PrincipalPropertySearch.allowed_children[(calendarserver_namespace, "limit")] = (0, 1)
 PrincipalPropertySearch.allowed_attributes["type"] = False
 Match.allowed_attributes = {

Deleted: CalendarServer/trunk/twistedcaldav/notify.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/notify.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/notify.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -1,1554 +0,0 @@
-# -*- test-case-name: twistedcaldav.test.test_notify -*-
-##
-# Copyright (c) 2005-2013 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-Notification framework for Calendar Server
-
-This module implements client code which is executed within the context of
-icalserver itself, and also server code (the "notification server") which is
-run as a separate process, launched as part of "./run".
-
-The notification server process is implemented as a twistd plugin
-(with a tapname of "caldav_notifier"), and is comprised of two
-services -- one handling the internal channel between icalserver
-and notification server, the other handling the external channel
-between notification server and a remote consumer.
-"""
-
-# TODO: add CalDAVTester test for examining new xmpp-uri property
-
-import uuid
-from fnmatch import fnmatch
-
-from zope.interface import Interface, implements
-
-from twext.python.log import LoggingMixIn, Logger
-
-from twisted.internet.protocol import ReconnectingClientFactory, ServerFactory
-from twisted.internet.ssl import ClientContextFactory
-from twisted.internet.defer import inlineCallbacks, Deferred, returnValue
-from twisted.protocols.basic import LineReceiver
-from twisted.plugin import IPlugin
-from twisted.application import internet, service
-from twisted.python.usage import Options, UsageError
-from twisted.python.reflect import namedClass
-from twisted.words.protocols.jabber import xmlstream
-from twisted.words.protocols.jabber.jid import JID
-from twisted.words.protocols.jabber.client import XMPPAuthenticator, IQAuthInitializer
-from twisted.words.protocols.jabber.xmlstream import IQ
-from twisted.words.xish import domish
-from twistedcaldav.config import config
-from twistedcaldav.memcacher import Memcacher
-from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
-from twistedcaldav import memcachepool
-from twext.internet.gaiendpoint import GAIEndpoint
-from twext.internet.adaptendpoint import connect
-
-log = Logger()
-
-__all__ = [
-    "Coalescer",
-    "INotifier",
-    "InternalNotificationFactory",
-    "InternalNotificationProtocol",
-    "NotificationClientFactory",
-    "NotificationClientLineProtocol",
-    "NotificationServiceMaker",
-    "Notifier",
-    "NotifierFactory",
-    "SimpleLineNotificationFactory",
-    "SimpleLineNotificationProtocol",
-    "SimpleLineNotifier",
-    "SimpleLineNotifierService",
-    "XMPPNotificationFactory",
-    "XMPPNotifier",
-    "getNodeCacher",
-    "getPubSubAPSConfiguration",
-    "getPubSubConfiguration",
-    "getPubSubHeartbeatURI",
-    "getPubSubPath",
-    "getPubSubXMPPURI",
-    "getXMPPSettings",
-]
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Classes used within calendarserver itself
-#
-
-class Notifier(LoggingMixIn):
-    """
-    Provides a hook for sending change notifications to the
-    L{NotifierFactory}.
-    """
-
-    def __init__(self, notifierFactory, label="default", id=None, prefix=None):
-        self._notifierFactory = notifierFactory
-        self._ids = { label : self.normalizeID(id) }
-        self._notify = True
-        self._prefix = prefix
-
-    def normalizeID(self, id):
-        urn = "urn:uuid:"
-        try:
-            if id.startswith(urn):
-                return id[len(urn):]
-        except AttributeError:
-            pass
-        return id
-
-    def enableNotify(self, arg):
-        self.log_debug("enableNotify: %s" % (self._ids['default'][1],))
-        self._notify = True
-
-    def disableNotify(self):
-        self.log_debug("disableNotify: %s" % (self._ids['default'][1],))
-        self._notify = False
-
-    def notify(self, op="update"):
-        for label in self._ids.iterkeys():
-            id = self.getID(label=label)
-            if id is not None:
-                if self._notify:
-                    self.log_debug("Notifications are enabled: %s %s %s" %
-                        (op, label, id))
-                    self._notifierFactory.send(op, id)
-                else:
-                    self.log_debug("Skipping notification for: %s" % (id,))
-
-    def clone(self, label="default", id=None):
-        newNotifier = self.__class__(self._notifierFactory)
-        newNotifier._ids = self._ids.copy()
-        newNotifier._ids[label] = id
-        newNotifier._prefix = self._prefix
-        return newNotifier
-
-    def addID(self, label="default", id=None):
-        self._ids[label] = self.normalizeID(id)
-
-    def getID(self, label="default"):
-        id = self._ids.get(label, None)
-        if self._prefix is None:
-            return id
-        else:
-            return "%s|%s" % (self._prefix, id)
-
-    @inlineCallbacks
-    def nodeName(self, label="default"):
-        id = self.getID(label=label)
-        pubSubConfig = self._notifierFactory.pubSubConfig
-        name = getPubSubPath(id, pubSubConfig)
-        if pubSubConfig["enabled"]:
-            try:
-                if self._notifierFactory.nodeCacher:
-                    nodeCacher = self._notifierFactory.nodeCacher
-                else:
-                    nodeCacher = getNodeCacher()
-                (yield nodeCacher.waitForNode(self, name))
-            except NodeCreationException, e:
-                self.log_warn(e)
-                returnValue(None)
-        returnValue(name)
-
-class NotificationClientLineProtocol(LineReceiver, LoggingMixIn):
-    """
-    Notification Client Line Protocol
-
-    Sends updates to the notification server.
-    """
-
-    def connectionMade(self):
-        self.client.addObserver(self)
-        self.factory.connectionMade()
-
-    def connectionLost(self, reason):
-        self.client.removeObserver(self)
-
-
-class NotificationClientFactory(ReconnectingClientFactory,
-    LoggingMixIn):
-    """
-    Notification Client Factory
-
-    Sends updates to the notification server.
-    """
-
-    protocol = NotificationClientLineProtocol
-
-    def __init__(self, client):
-        self.connected = False
-        self.client = client
-
-    def clientConnectionLost(self, connector, reason):
-        self.log_error("Connect to notification server lost: %s" % (reason,))
-        self.connected = False
-        ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
-
-    def clientConnectionFailed(self, connector, reason):
-        self.log_error("Unable to connect to notification server: %s" % (reason,))
-        self.connected = False
-        ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
-
-    def connectionMade(self):
-        self.connected = True
-        self.resetDelay()
-        self.client.connectionMade()
-
-    def isReady(self):
-        return self.connected
-
-    def buildProtocol(self, addr):
-        p = self.protocol()
-        p.factory = self
-        p.client = self.client
-        return p
-
-
-class NotifierFactory(LoggingMixIn):
-    """
-    Notifier Factory
-
-    Creates Notifier instances and forwards notifications from them to the
-    gateway.
-    """
-
-    def __init__(self, gatewayHost, gatewayPort, pubSubConfig=None,
-        nodeCacher=None, reactor=None):
-
-        self.factory = None
-
-        self.gatewayHost = gatewayHost
-        self.gatewayPort = gatewayPort
-        self.pubSubConfig = pubSubConfig
-        self.nodeCacher = nodeCacher
-
-        self.observers = set()
-        self.queued = set()
-
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-
-    def send(self, op, id):
-        if self.factory is None:
-            self.factory = NotificationClientFactory(self)
-            connect(
-                GAIEndpoint(self.reactor, self.gatewayHost, self.gatewayPort),
-                self.factory)
-            self.log_debug("Creating factory")
-
-        msg = "%s %s" % (op, str(id))
-        if self.factory.isReady() and self.observers:
-            for observer in self.observers:
-                self.log_debug("Sending to notification server: %s" % (msg,))
-                observer.sendLine(msg)
-        else:
-            self.log_debug("Queuing: %s" % (msg,))
-            self.queued.add(msg)
-
-    def connectionMade(self):
-        if self.factory.isReady() and self.observers:
-            for observer in self.observers:
-                for msg in self.queued:
-                    self.log_debug("Sending from queue: %s" % (msg,))
-                    observer.sendLine(msg)
-            self.queued.clear()
-
-    def addObserver(self, observer):
-        self.observers.add(observer)
-
-    def removeObserver(self, observer):
-        self.observers.remove(observer)
-
-    def newNotifier(self, label="default", id=None, prefix=None):
-        return Notifier(self, label=label, id=id, prefix=prefix)
-
-
-
-
-class NodeCreationException(Exception):
-    pass
-
-class NodeCacher(Memcacher, LoggingMixIn):
-
-    def __init__(self, reactor=None):
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-        super(NodeCacher, self).__init__("pubsubnodes")
-
-    def nodeExists(self, nodeName):
-        return self.get(nodeName)
-
-    def storeNode(self, nodeName):
-        return self.set(nodeName, "1")
-
-    @inlineCallbacks
-    def waitForNode(self, notifier, nodeName):
-        retryCount = 0
-        verified = False
-        requestedCreation = False
-        while(retryCount < 5):
-            if (yield self.nodeExists(nodeName)):
-                verified = True
-                break
-
-            if not requestedCreation:
-                notifier.notify(op="create")
-                requestedCreation = True
-
-            retryCount += 1
-
-            pause = Deferred()
-            def _timedDeferred():
-                pause.callback(True)
-            self.reactor.callLater(1, _timedDeferred)
-            yield pause
-
-        if not verified:
-            self.log_debug("Giving up!")
-            raise NodeCreationException("Could not create node %s" % (nodeName,))
-
-    def createNode(self, notifier, nodeName):
-        """
-        Check with memcached to see if this node is known to exist, and if
-        not, request it be created (without waiting)
-        """
-        def _nodeExistenceChecked(result):
-            if not result:
-                notifier.notify(op="create")
-
-        d = self.nodeExists(nodeName)
-        d.addCallback(_nodeExistenceChecked)
-        return d
-
-
-_nodeCacher = None
-
-def getNodeCacher():
-    global _nodeCacher
-    if _nodeCacher is None:
-        _nodeCacher = NodeCacher()
-    return _nodeCacher
-
-
-
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Classes used within Notification Server
-#
-
-#
-# Internal Channel (from icalserver to notification server)
-#
-
-class InternalNotificationProtocol(LineReceiver):
-    """
-    InternalNotificationProtocol
-
-    Receives notifications from the calendar server.
-    """
-
-    def lineReceived(self, line):
-        try:
-            op, id = line.strip().split()
-            self.factory.coalescer.add(op, id)
-        except ValueError:
-            # ignore invalid input
-            log.error("Invalid input received on internal notification port: %s"
-                % (line,))
-
-
-class InternalNotificationFactory(ServerFactory):
-    """
-    Internal Notification Factory
-
-    Receives notifications from the calendar server.
-    """
-
-    protocol = InternalNotificationProtocol
-
-    def __init__(self, notifiers, delaySeconds=None):
-        self.coalescer = Coalescer(notifiers, delaySeconds=delaySeconds)
-
-
-
-class Coalescer(LoggingMixIn):
-    """
-    Coalescer
-
-    A queue which hangs on to incoming ids for some period of time before
-    passing them along to the external notifier listening for these updates.
-    A chatty CalDAV client can make several changes in a short period of time,
-    and the Coalescer buffers the external clients somewhat.
-    """
-
-    delaySeconds = 5
-
-    # sendAnywayAfterCount can be used to control how many times a notification
-    # countdown timer is reset because of new changes.  Once a notification
-    # has been delayed 'sendAnywayAfterCount' times, it is sent anyway,
-    # otherwise a busy calendar might never have a notification sent out.
-    # Set this to 0 to disable the timer reset feature.
-    sendAnywayAfterCount = 0
-
-    def __init__(self, notifiers, reactor=None, delaySeconds=None,
-        sendAnywayAfterCount=None):
-
-        if sendAnywayAfterCount:
-            self.sendAnywayAfterCount = sendAnywayAfterCount
-
-        if delaySeconds is not None:
-            self.delaySeconds = delaySeconds
-
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-
-        self.ids = {}
-        self.notifiers = notifiers
-
-    def add(self, op, id):
-
-        if op == "create":
-            # we don't want to delay a "create" notification; this opcode
-            # is meant for XMPP pubsub -- it means create and configure the
-            # node but don't publish to it
-            for notifier in self.notifiers:
-                notifier.enqueue(op, id)
-
-        else: # normal update notification
-            delayed, count = self.ids.get(id, [None, 0])
-
-            if delayed and delayed.active():
-                count += 1
-                if count < self.sendAnywayAfterCount:
-                    # reschedule for delaySeconds in the future
-                    delayed.reset(self.delaySeconds)
-                    self.ids[id][1] = count
-                    self.log_debug("Delaying: %s" % (id,))
-                else:
-                    self.log_debug("Not delaying to avoid starvation: %s" % (id,))
-            else:
-                self.log_debug("Scheduling: %s" % (id,))
-                self.ids[id] = [self.reactor.callLater(self.delaySeconds,
-                    self.delayedEnqueue, op, id), 0]
-
-    def delayedEnqueue(self, op, id):
-        self.log_debug("Time to send: %s" % (id,))
-        self.ids[id][1] = 0
-        for notifier in self.notifiers:
-            notifier.enqueue(op, id)
-
-
-
-#
-# External Channel (from notification server to other consumers)
-#
-
-class INotifier(Interface):
-    """
-    Notifier Interface
-
-    Defines an enqueue method that Notifier classes need to implement.
-    """
-
-    def enqueue(self, op, id):
-        """
-        Let the notifier object know that a change has been made for this
-        id, and enough time has passed to allow for coalescence.
-
-        @type op: C{str}
-        @type id: C{str}
-        """
-
-
-class SimpleLineNotifier(LoggingMixIn):
-    """
-    Simple Line Notifier
-
-    Listens for ids from the coalescer and writes them out to any
-    connected clients.  Each line is simply a sequence number, a
-    space, and an id string.  If the external client sends a sequence
-    number, this notifier will send notification lines for each id
-    that was changed since that sequence number was originally sent.
-    A history of such sequence numbers is stored in a python dict.
-    If the external client sends a zero, then the history is cleared
-    and the next sequence number to use is reset to 1.
-
-    The sequence number is stored as a python long which means it has
-    essentially infinite precision.  We discussed rolling over at the
-    64-bit boundary, but even if we limit the sequence number to a 64-bit
-    signed integer (or 2^63), and we had 100,000 users generating the
-    maximum number of notifications (which by default is 6/minute since
-    we're coalescing over 10 seconds), it would take 29 million years to
-    rollover.
-    """
-
-    implements(INotifier)
-
-    def __init__(self, settings):
-        self.reset()
-        self.observers = set()
-        self.sentReset = False
-
-    def enqueue(self, op, id):
-
-        if op == "update":
-
-            self.latestSeq += 1L
-
-            # Update history
-            self.history[id] = self.latestSeq
-
-            for observer in self.observers:
-                msg = "%d %s" % (self.latestSeq, id)
-                self.log_debug("Sending %s" % (msg,))
-                observer.sendLine(msg)
-
-    def reset(self):
-        self.latestSeq = 0L
-        self.history = { } # keys=id, values=sequenceNumber
-
-    def playback(self, observer, oldSeq):
-
-        hist = self.history
-        toSend = [(hist[id], id) for id in hist if hist[id] > oldSeq]
-        toSend.sort() # sorts the tuples based on numeric sequence number
-
-        for seq, id in toSend:
-            msg = "%d %s" % (seq, id)
-            self.log_debug("Sending %s" % (msg,))
-            observer.sendLine(msg)
-
-
-    def addObserver(self, observer):
-        self.observers.add(observer)
-
-    def removeObserver(self, observer):
-        self.observers.remove(observer)
-
-    def connectionMade(self, observer):
-        if not self.sentReset:
-            self.log_debug("Sending 0")
-            observer.sendLine("0")
-            self.sentReset = True
-
-
-class SimpleLineNotificationProtocol(LineReceiver, LoggingMixIn):
-    """
-    Simple Line Notification Protocol
-
-    Sends notifications to external consumers.  Also responds to history-
-    playback requests.  If an integer is received from an external consumer,
-    it is interpreted as a sequence number; all notifications sent since that
-    sequence number was sent are resent.
-    """
-
-    def connectionMade(self):
-        # we just received a connection from the outside; if it's the first
-        # since we started running, it means we need to let them know that
-        # a reset has happened.  This assumes that only one connection will
-        # be made to this channel; if we end up having multiple consumers
-        # of this protocol, we would need to uniquely identify them.
-        self.notifier.connectionMade(self)
-
-    def lineReceived(self, line):
-        val = line.strip()
-
-        # Should be a number requesting all updates since that sequence
-        try:
-            oldSeq = int(val)
-        except ValueError, e:
-            self.log_warn("Error parsing %s: %s (from %s)" % (val, e,
-                self.transport.getPeer()))
-            return
-
-        if oldSeq == 0:
-            self.notifier.reset()
-        else:
-            self.notifier.playback(self, oldSeq)
-
-    def connectionLost(self, reason):
-        self.notifier.removeObserver(self)
-
-
-class SimpleLineNotificationFactory(ServerFactory):
-    """
-    Simple Line Notification Factory
-
-    Sends notifications to external consumers.
-    """
-
-    protocol = SimpleLineNotificationProtocol
-
-    def __init__(self, notifier):
-        self.notifier = notifier
-
-    def buildProtocol(self, addr):
-        p = self.protocol()
-        self.notifier.addObserver(p)
-        p.notifier = self.notifier
-        return p
-
-
-
-
-
-
-class XMPPNotifier(LoggingMixIn):
-    """
-    XMPP Notifier
-
-    Uses pubsub XMPP requests to let subscribers know when there
-    has been a change made to a DAV resource (currently just
-    CalendarHomeResources).  Uses XMPP login info from the config file
-    to determine which pubsub service to connect to.  When it's
-    time to send a notification, XMPPNotifier computes a node path
-    corresponding to the DAV resource and emits a publish request
-    for that node.  If the request comes back 404 XMPPNotifier will
-    create the node and then go through the configuration process,
-    followed by a publish retry.
-
-    For monitoring purposes, you can subscribe to the server's JID
-    as long as your own JID matches the "AllowedJIDs" pattern(s) in
-    the config file; XMPPNotifier will send error messages to your
-    JID.  If you also want to receive non-error, debug messages,
-    send the calendar server JID the message, "debug on".  Send
-    "help" for other commands.
-
-    To let clients know that the notifications from the calendar server
-    are still flowing, a "heartbeat" node is published to every 30
-    minutes (configurable).
-
-    """
-
-    implements(INotifier)
-
-    pubsubNS = 'http://jabber.org/protocol/pubsub'
-
-    def __init__(self, settings, reactor=None, configOverride=None,
-        heartbeat=True, roster=True):
-        self.xmlStream = None
-        self.settings = settings
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-        self.config = configOverride or config
-        self.doHeartbeat = heartbeat and self.settings['HeartbeatMinutes'] != 0
-        self.doRoster = roster
-
-        self.roster = {}
-        self.outstanding = {}
-
-    def lockNode(self, nodeName):
-        if self.outstanding.has_key(nodeName):
-            return False
-        else:
-            self.outstanding[nodeName] = 1
-            return True
-
-    def unlockNode(self, failure, nodeName):
-        try:
-            del self.outstanding[nodeName]
-        except KeyError:
-            pass
-
-    def sendHeartbeat(self):
-        if self.doHeartbeat and self.xmlStream is not None:
-            self.enqueue("update", "", lock=False)
-            self.reactor.callLater(self.settings['HeartbeatMinutes'] * 60,
-                self.sendHeartbeat)
-
-    def enqueue(self, op, id, lock=True):
-        if self.xmlStream is not None:
-            # Convert id to node
-            nodeName = getPubSubPath(id, getPubSubConfiguration(self.config))
-            if op == "create":
-                if not self.lockNode(nodeName):
-                    # this node is busy, so it must already be created, or at
-                    # least in the proccess
-                    return
-                self.createNode(nodeName, publish=False)
-            else:
-                self.publishNode(nodeName, lock=lock)
-
-    def publishNode(self, nodeName, lock=True):
-        if self.xmlStream is None:
-            # We lost our connection
-            self.unlockNode(None, nodeName)
-            return
-
-        try:
-            if lock and not self.lockNode(nodeName):
-                return
-
-            iq = IQ(self.xmlStream)
-            pubsubElement = iq.addElement('pubsub', defaultUri=self.pubsubNS)
-            publishElement = pubsubElement.addElement('publish')
-            publishElement['node'] = nodeName.decode("utf-8")
-            if self.settings["NodeConfiguration"]["pubsub#deliver_payloads"] == '1':
-                itemElement = publishElement.addElement('item')
-                itemElement.addElement('plistfrag', defaultUri='plist-apple')
-
-            self.sendDebug("Publishing (%s)" % (nodeName,), iq)
-            d = iq.send(to=self.settings['ServiceAddress'])
-            d.addCallback(self.publishNodeSuccess, nodeName)
-            d.addErrback(self.publishNodeFailure, nodeName)
-        except:
-            self.unlockNode(None, nodeName)
-            raise
-
-    def publishNodeSuccess(self, iq, nodeName):
-        self.unlockNode(None, nodeName)
-        self.sendDebug("Node publish successful (%s)" % (nodeName,), iq)
-
-    def publishNodeFailure(self, result, nodeName):
-        try:
-            iq = result.value.getElement()
-
-            if iq.name == "error":
-                if iq['code'] == '400':
-                    self.requestConfigurationForm(nodeName, True)
-
-                elif iq['code'] == '404':
-                    self.createNode(nodeName)
-            else:
-                self.log_error("PubSub node publish error: %s" %
-                    (iq.toXml().encode('ascii', 'replace')),)
-                self.sendDebug("Node publish failed (%s)" % (nodeName,), iq)
-                # Don't know how to proceed
-                self.unlockNode(None, nodeName)
-        except AttributeError:
-            # We did not get an XML response; most likely it was a disconnection
-            self.unlockNode(None, nodeName)
-            # Don't re-raise, just unlock and ignore
-        except:
-            # Note: this block is not a "finally" because in the case of a 404
-            # we don't want to unlock yet
-            self.unlockNode(None, nodeName)
-            raise
-
-    def createNode(self, nodeName, publish=True):
-        if self.xmlStream is None:
-            # We lost our connection
-            self.unlockNode(None, nodeName)
-            return
-
-        try:
-            iq = IQ(self.xmlStream)
-            pubsubElement = iq.addElement('pubsub', defaultUri=self.pubsubNS)
-            child = pubsubElement.addElement('create')
-            child['node'] = nodeName.decode("utf-8")
-            d = iq.send(to=self.settings['ServiceAddress'])
-            d.addCallback(self.createNodeSuccess, nodeName, publish)
-            d.addErrback(self.createNodeFailure, nodeName, publish)
-        except:
-            self.unlockNode(None, nodeName)
-            raise
-
-    def createNodeSuccess(self, iq, nodeName, publish):
-        try:
-            self.sendDebug("Node creation successful (%s)" % (nodeName,), iq)
-            # now time to configure; fetch the form
-            self.requestConfigurationForm(nodeName, publish)
-        except:
-            self.unlockNode(None, nodeName)
-            raise
-
-    def createNodeFailure(self, result, nodeName, publish):
-        try:
-            iq = result.value.getElement()
-            if iq['code'] == '409':
-                # node already exists, proceed to configure
-                self.sendDebug("Node already exists (%s)" % (nodeName,), iq)
-                self.requestConfigurationForm(nodeName, publish)
-            else:
-                # couldn't create node, give up
-                self.unlockNode(None, nodeName)
-                self.log_error("PubSub node creation error: %s" %
-                    (iq.toXml().encode('ascii', 'replace')),)
-                self.sendError("Node creation failed (%s)" % (nodeName,), iq)
-        except AttributeError:
-            # We did not get an XML response; most likely it was a disconnection
-            self.unlockNode(None, nodeName)
-            # Don't re-raise, just unlock and ignore
-        except:
-            # Note: this block is not a "finally" because in the case of a 409
-            # we don't want to unlock yet
-            self.unlockNode(None, nodeName)
-            raise
-
-    def requestConfigurationForm(self, nodeName, publish):
-        if self.xmlStream is None:
-            # We lost our connection
-            self.unlockNode(None, nodeName)
-            return
-
-        try:
-            # XXX This codepath is not unit tested
-            iq = IQ(self.xmlStream, 'get')
-            child = iq.addElement('pubsub',
-                defaultUri=self.pubsubNS+"#owner")
-            child = child.addElement('configure')
-            child['node'] = nodeName.decode("utf-8")
-            d = iq.send(to=self.settings['ServiceAddress'])
-            d.addCallback(self.requestConfigurationFormSuccess, nodeName,
-                publish)
-            d.addErrback(self.requestConfigurationFormFailure, nodeName)
-        except:
-            self.unlockNode(None, nodeName)
-            raise
-
-    def _getChild(self, element, name):
-        for child in element.elements():
-            if child.name == name:
-                return child
-        return None
-
-    def requestConfigurationFormSuccess(self, iq, nodeName, publish):
-        if self.xmlStream is None:
-            # We lost our connection
-            self.unlockNode(None, nodeName)
-            return
-
-        try:
-            nodeConf = self.settings["NodeConfiguration"]
-            self.sendDebug("Received configuration form (%s)" % (nodeName,), iq)
-            pubsubElement = self._getChild(iq, 'pubsub')
-            if pubsubElement:
-                configureElement = self._getChild(pubsubElement, 'configure')
-                if configureElement:
-                    formElement = configureElement.firstChildElement()
-                    if formElement['type'] == 'form':
-                        # We've found the form; start building a response
-                        filledIq = IQ(self.xmlStream, 'set')
-                        filledPubSub = filledIq.addElement('pubsub',
-                            defaultUri=self.pubsubNS+"#owner")
-                        filledConfigure = filledPubSub.addElement('configure')
-                        filledConfigure['node'] = nodeName.decode("utf-8")
-                        filledForm = filledConfigure.addElement('x',
-                            defaultUri='jabber:x:data')
-                        filledForm['type'] = 'submit'
-
-                        configMatches = True
-                        for field in formElement.elements():
-                            if field.name == 'field':
-                                var = field['var']
-                                if var == "FORM_TYPE":
-                                    filledForm.addChild(field)
-                                else:
-                                    value = nodeConf.get(var, None)
-                                    if (value is not None and
-                                        (str(self._getChild(field,
-                                        "value")) != value)):
-                                        # this field needs configuring
-                                        configMatches = False
-                                        filledField = filledForm.addElement('field')
-                                        filledField['var'] = var
-                                        filledField['type'] = field['type']
-                                        valueElement = filledField.addElement('value')
-                                        valueElement.addContent(value)
-                                        # filledForm.addChild(field)
-                        if configMatches:
-                            # XXX This codepath is not unit tested
-                            cancelIq = IQ(self.xmlStream, 'set')
-                            cancelPubSub = cancelIq.addElement('pubsub',
-                                defaultUri=self.pubsubNS+"#owner")
-                            cancelConfig = cancelPubSub.addElement('configure')
-                            cancelConfig['node'] = nodeName.decode("utf-8")
-                            cancelX = cancelConfig.addElement('x',
-                                defaultUri='jabber:x:data')
-                            cancelX['type'] = 'cancel'
-                            self.sendDebug("Cancelling configuration (%s)"
-                                           % (nodeName,), cancelIq)
-                            d = cancelIq.send(to=self.settings['ServiceAddress'])
-                        else:
-                            self.sendDebug("Sending configuration form (%s)"
-                                           % (nodeName,), filledIq)
-                            d = filledIq.send(to=self.settings['ServiceAddress'])
-                        d.addCallback(self.configurationSuccess, nodeName,
-                            publish)
-                        d.addErrback(self.configurationFailure, nodeName)
-                        return
-
-            # Couldn't process configuration form, give up
-            self.unlockNode(None, nodeName)
-
-        except:
-            # Couldn't process configuration form, give up
-            self.unlockNode(None, nodeName)
-            raise
-
-    def requestConfigurationFormFailure(self, result, nodeName):
-        # If we get here we're giving up
-        try:
-            iq = result.value.getElement()
-            self.log_error("PubSub configuration form request error: %s" %
-                (iq.toXml().encode('ascii', 'replace')),)
-            self.sendError("Failed to receive configuration form (%s)" %
-                (nodeName,), iq)
-        except AttributeError:
-            # We did not get an XML response; most likely it was a disconnection
-            pass
-        finally:
-            self.unlockNode(None, nodeName)
-
-    def configurationSuccess(self, iq, nodeName, publish):
-        if self.xmlStream is None:
-            # We lost our connection
-            self.unlockNode(None, nodeName)
-            return
-
-        try:
-            self.log_debug("PubSub node %s is configured" % (nodeName,))
-            self.sendDebug("Configured node (%s)" % (nodeName,), iq)
-            nodeCacher = getNodeCacher()
-            nodeCacher.storeNode(nodeName)
-            if publish:
-                self.publishNode(nodeName, lock=False)
-            else:
-                self.unlockNode(None, nodeName)
-        except:
-            self.unlockNode(None, nodeName)
-            raise
-
-    def configurationFailure(self, result, nodeName):
-        # If we get here we're giving up
-        try:
-            iq = result.value.getElement()
-            self.log_error("PubSub node configuration error: %s" %
-                (iq.toXml().encode('ascii', 'replace')),)
-            self.sendError("Failed to configure node (%s)" % (nodeName,), iq)
-        except AttributeError:
-            # We did not get an XML response; most likely it was a disconnection
-            pass
-        finally:
-            self.unlockNode(None, nodeName)
-
-    def deleteNode(self, nodeName):
-        if self.xmlStream is None:
-            # We lost our connection
-            self.unlockNode(None, nodeName)
-            return
-
-        try:
-            if not self.lockNode(nodeName):
-                return
-
-            iq = IQ(self.xmlStream)
-            pubsubElement = iq.addElement('pubsub',
-                defaultUri=self.pubsubNS+"#owner")
-            publishElement = pubsubElement.addElement('delete')
-            publishElement['node'] = nodeName.decode("utf-8")
-            self.sendDebug("Deleting (%s)" % (nodeName,), iq)
-            d = iq.send(to=self.settings['ServiceAddress'])
-            d.addCallback(self.deleteNodeSuccess, nodeName)
-            d.addErrback(self.deleteNodeFailure, nodeName)
-        except:
-            self.unlockNode(None, nodeName)
-            raise
-
-    def deleteNodeSuccess(self, iq, nodeName):
-        self.unlockNode(None, nodeName)
-        self.sendDebug("Node delete successful (%s)" % (nodeName,), iq)
-
-    def deleteNodeFailure(self, result, nodeName):
-        try:
-            iq = result.value.getElement()
-            self.log_error("PubSub node delete error: %s" %
-                (iq.toXml().encode('ascii', 'replace')),)
-            self.sendDebug("Node delete failed (%s)" % (nodeName,), iq)
-        except AttributeError:
-            # We did not get an XML response; most likely it was a disconnection
-            pass
-        finally:
-            self.unlockNode(None, nodeName)
-
-
-    def requestRoster(self):
-        if self.doRoster:
-            self.roster = {}
-            rosterIq = IQ(self.xmlStream, 'get')
-            rosterIq.addElement("query", "jabber:iq:roster")
-            d = rosterIq.send()
-            d.addCallback(self.handleRoster)
-
-    def allowedInRoster(self, jid):
-        """ Returns True if jid matches any of the patterns in AllowedJIDs,
-            or is our own JID.  False otherwise. """
-
-        # Always allow our own JID (in case multiple servers are sharing it)
-        settings = self.settings
-        if settings is not None:
-            if settings["JID"] == jid:
-                return True
-
-        for pattern in self.settings.get("AllowedJIDs", []):
-            if fnmatch(jid, pattern):
-                return True
-        return False
-
-    def handleRoster(self, iq):
-        for child in iq.children[0].children:
-            jid = child['jid']
-            if self.allowedInRoster(jid):
-                self.log_debug("In roster: %s" % (jid,))
-                if not self.roster.has_key(jid):
-                    self.roster[jid] = { 'debug' : False, 'available' : False }
-            else:
-                self.log_info("JID not allowed in roster: %s" % (jid,))
-
-    def handlePresence(self, iq):
-        self.log_debug("Presence IQ: %s" %
-            (iq.toXml().encode('ascii', 'replace')),)
-        presenceType = iq.getAttribute('type')
-
-        if presenceType == 'subscribe':
-            frm = JID(iq['from']).userhost()
-            if self.allowedInRoster(frm):
-                self.roster[frm] = { 'debug' : False, 'available' : True }
-                response = domish.Element(('jabber:client', 'presence'))
-                response['to'] = iq['from']
-                response['type'] = 'subscribed'
-                self.xmlStream.send(response)
-
-                # request subscription as well
-                subscribe = domish.Element(('jabber:client', 'presence'))
-                subscribe['to'] = iq['from']
-                subscribe['type'] = 'subscribe'
-                self.xmlStream.send(subscribe)
-            else:
-                self.log_info("JID not allowed in roster: %s" % (frm,))
-                # Reject
-                response = domish.Element(('jabber:client', 'presence'))
-                response['to'] = iq['from']
-                response['type'] = 'unsubscribed'
-                self.xmlStream.send(response)
-
-        elif presenceType == 'unsubscribe':
-            frm = JID(iq['from']).userhost()
-            if self.roster.has_key(frm):
-                del self.roster[frm]
-            response = domish.Element(('jabber:client', 'presence'))
-            response['to'] = iq['from']
-            response['type'] = 'unsubscribed'
-            self.xmlStream.send(response)
-
-            # remove from roster as well
-            # XXX This codepath is not unit tested
-            removal = IQ(self.xmlStream, 'set')
-            query = removal.addElement("query", "jabber:iq:roster")
-            query.addElement("item")
-            query.item["jid"] = iq["from"]
-            query.item["subscription"] = "remove"
-            removal.send()
-
-        elif presenceType == 'unavailable':
-            frm = JID(iq['from']).userhost()
-            if self.roster.has_key(frm):
-                self.roster[frm]['available'] = False
-
-        else:
-            frm = JID(iq['from']).userhost()
-            if self.allowedInRoster(frm):
-                if self.roster.has_key(frm):
-                    self.roster[frm]['available'] = True
-                else:
-                    self.roster[frm] = { 'debug' : False, 'available' : True }
-            else:
-                self.log_info("JID not allowed in roster: %s" % (frm,))
-
-    def streamOpened(self, xmlStream):
-        self.xmlStream = xmlStream
-        xmlStream.addObserver('/message', self.handleMessage)
-        xmlStream.addObserver('/presence', self.handlePresence)
-        self.requestRoster()
-        self.sendHeartbeat()
-
-
-    def streamClosed(self):
-        self.xmlStream = None
-
-    def sendDebug(self, txt, element):
-        txt = "DEBUG: %s %s" % (txt, element.toXml().encode('ascii', 'replace'))
-        for jid, info in self.roster.iteritems():
-            if info['available'] and info['debug']:
-                self.sendAlert(jid, txt)
-
-    def sendError(self, txt, element):
-        txt = "ERROR: %s %s" % (txt, element.toXml().encode('ascii', 'replace'))
-        for jid, info in self.roster.iteritems():
-            if info['available']:
-                self.sendAlert(jid, txt)
-
-    def sendAlert(self, jid, txt):
-        if self.xmlStream is not None:
-            message = domish.Element(('jabber:client', 'message'))
-            message['to'] = JID(jid).full()
-            message.addElement('body', content=txt)
-            self.xmlStream.send(message)
-
-    def handleMessage(self, iq):
-        body = getattr(iq, 'body', None)
-        if body:
-            response = None
-            frm = JID(iq['from']).userhost()
-            if frm in self.roster:
-                txt = str(body).lower()
-                if txt == "help":
-                    response = "debug on, debug off, roster, create <nodename>, publish <nodename>, hammer <count>"
-                elif txt == "roster":
-                    response = "Roster: %s" % (str(self.roster),)
-                elif txt == "debug on":
-                    self.roster[frm]['debug'] = True
-                    response = "Debugging on"
-                elif txt == "debug off":
-                    self.roster[frm]['debug'] = False
-                    response = "Debugging off"
-                elif txt == "outstanding":
-                    response = "Outstanding: %s" % (str(self.outstanding),)
-                elif txt.startswith("publish"):
-                    try:
-                        publish, nodeName = str(body).split()
-                    except ValueError:
-                        response = "Please phrase it like 'publish nodename'"
-                    else:
-                        response = "Publishing node %s" % (nodeName,)
-                        self.reactor.callLater(1, self.enqueue, "update",
-                            nodeName)
-                elif txt.startswith("delete"):
-                    try:
-                        delete, nodeName = str(body).split()
-                    except ValueError:
-                        response = "Please phrase it like 'delete nodename'"
-                    else:
-                        response = "Deleting node %s" % (nodeName,)
-                        self.reactor.callLater(1, self.deleteNode, nodeName)
-                elif txt.startswith("create"):
-                    try:
-                        publish, nodeName = str(body).split()
-                    except ValueError:
-                        response = "Please phrase it like 'create nodename'"
-                    else:
-                        response = "Creating and configuring node %s" % (nodeName,)
-                        self.reactor.callLater(1, self.enqueue, "create",
-                            nodeName)
-                elif txt.startswith("hammer"):
-                    try:
-                        hammer, count = txt.split()
-                        count = int(count)
-                    except ValueError:
-                        response = "Please phrase it like 'hammer 100'"
-                    else:
-                        response = "Hammer will commence now, %d times" % (count,)
-                        self.reactor.callLater(1, self.hammer, count)
-                else:
-                    response = "I don't understand.  Try 'help'."
-            else:
-                response = "Sorry, you are not authorized to converse with this server"
-
-            if response:
-                message = domish.Element(('jabber:client', 'message'))
-                message['to'] = JID(iq['from']).full()
-                message.addElement('body', content=response)
-                self.xmlStream.send(message)
-
-
-    def hammer(self, count):
-        for i in xrange(count):
-            self.enqueue("update", "hammertesting%d" % (i,))
-
-
-class XMPPNotificationFactory(xmlstream.XmlStreamFactory, LoggingMixIn):
-
-    def __init__(self, notifier, settings, reactor=None, keepAlive=True):
-        self.log_warn("Setting up XMPPNotificationFactory")
-
-        self.notifier = notifier
-        self.settings = settings
-
-        self.jid = settings['JID']
-
-        # Ignore JID resource from plist
-        slash = self.jid.find("/")
-        if slash > -1:
-            self.jid = self.jid[0:slash]
-
-        # Generate a unique JID resource value
-        resource = "icalserver.%s" % uuid.uuid4().hex
-        self.jid = "%s/%s" % (self.jid, resource)
-
-        self.keepAliveSeconds = settings.get('KeepAliveSeconds', 120)
-        self.xmlStream = None
-        self.presenceCall = None
-        self.doKeepAlive = keepAlive
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-
-        xmlstream.XmlStreamFactory.__init__(self,
-            XMPPAuthenticator(JID(self.jid), settings['Password']))
-
-        self.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.connected)
-        self.addBootstrap(xmlstream.STREAM_END_EVENT, self.disconnected)
-        self.addBootstrap(xmlstream.INIT_FAILED_EVENT, self.initFailed)
-
-        self.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authenticated)
-        self.addBootstrap(IQAuthInitializer.INVALID_USER_EVENT,
-            self.authFailed)
-        self.addBootstrap(IQAuthInitializer.AUTH_FAILED_EVENT,
-            self.authFailed)
-
-    def connected(self, xmlStream):
-        self.xmlStream = xmlStream
-        self.log_warn("XMPP connection successful")
-        # Log all traffic
-        xmlStream.rawDataInFn = self.rawDataIn
-        xmlStream.rawDataOutFn = self.rawDataOut
-
-    def disconnected(self, xmlStream):
-        self.notifier.streamClosed()
-        self.xmlStream = None
-        if self.presenceCall is not None:
-            self.presenceCall.cancel()
-            self.presenceCall = None
-        self.log_warn("XMPP disconnected")
-
-    def initFailed(self, failure):
-        self.xmlStream = None
-        self.log_warn("XMPP Initialization failed: %s" % (failure,))
-
-    def authenticated(self, xmlStream):
-        self.log_warn("XMPP authentication successful: %s" % (self.jid,))
-        # xmlStream.addObserver('/message', self.handleMessage)
-        self.sendPresence()
-        self.notifier.streamOpened(xmlStream)
-
-    def authFailed(self, e):
-        self.log_error("Failed to log in XMPP (%s); check JID and password" %
-            (self.jid,))
-
-    def sendPresence(self):
-        if self.doKeepAlive and self.xmlStream is not None:
-            presence = domish.Element(('jabber:client', 'presence'))
-            self.xmlStream.send(presence)
-            self.presenceCall = self.reactor.callLater(self.keepAliveSeconds,
-                self.sendPresence)
-
-    def rawDataIn(self, buf):
-        self.log_debug("RECV: %s" % unicode(buf, 'utf-8').encode('ascii',
-            'replace'))
-
-    def rawDataOut(self, buf):
-        self.log_debug("SEND: %s" % unicode(buf, 'utf-8').encode('ascii',
-            'replace'))
-
-def getXMPPSettings(config):
-    """ Return the XMPP settings if both overall notifications are enabled
-        and XMPP is enabled; None otherwise.
-    """
-    if config.Notifications.Enabled:
-        # return the first enabled xmpp service settings in the config file
-        for key, settings in config.Notifications.Services.iteritems():
-            if (settings["Service"] == "twistedcaldav.notify.XMPPNotifierService"
-                and settings["Enabled"]):
-                return settings
-    return None
-
-def getPubSubConfiguration(config):
-    # TODO: Should probably cache this
-    results = { 'enabled' : False, 'host' : config.ServerHostName }
-    settings = getXMPPSettings(config)
-    if settings is not None:
-        results['enabled'] = True
-        results['service'] = settings['ServiceAddress']
-        results['port'] = config.SSLPort or config.HTTPPort
-        results['xmpp-server'] = (
-            settings['Host'] if settings['Port'] == 5222
-            else "%s:%d" % (settings['Host'], settings['Port'])
-        )
-        results['heartrate'] = settings['HeartbeatMinutes']
-
-    return results
-
-def getPubSubAPSConfiguration(id, config):
-    """
-    Returns the Apple push notification settings specific to the notifier
-    ID, which includes a prefix that is either "CalDAV" or "CardDAV"
-    """
-    try:
-        prefix, id = id.split("|", 1)
-    except ValueError:
-        # id has no prefix, so we can't look up APS config
-        return None
-
-    # If we are directly talking to apple push, advertise those settings
-    applePushSettings = config.Notifications.Services.ApplePushNotifier
-    if applePushSettings.Enabled:
-        settings = {}
-        settings["APSBundleID"] = applePushSettings[prefix]["Topic"]
-        if config.EnableSSL:
-            url = "https://%s:%s/%s" % (config.ServerHostName, config.SSLPort,
-                applePushSettings.SubscriptionURL)
-        else:
-            url = "http://%s:%s/%s" % (config.ServerHostName, config.HTTPPort,
-                applePushSettings.SubscriptionURL)
-        settings["SubscriptionURL"] = url
-        settings["SubscriptionRefreshIntervalSeconds"] = applePushSettings.SubscriptionRefreshIntervalSeconds
-        settings["APSEnvironment"] = applePushSettings.Environment
-        return settings
-
-    # ...otherwise pick up the apple push settings we get via XMPP and
-    # apn bridge
-    settings = getXMPPSettings(config)
-    if settings is None:
-        return None
-
-    if (settings.has_key(prefix) and
-        settings[prefix]["APSBundleID"] and
-        settings[prefix]["SubscriptionURL"]):
-        return settings[prefix]
-
-    return None
-
-
-
-
-def getPubSubPath(id, pubSubConfiguration):
-    """
-    Generate a pubsub node path from an id and the pubsub configuration
-    @param id: a string identifying the resource that was modified.  If
-        the id has a "|" in it, what is to the left of the first "|" is
-        treated as a prefix and will be used for the root of the path.
-    @type id: C{str}
-
-    @param pubSubConfiguration: a dictionary containing various relevant
-        configuration data
-    @type pubSubConfiguration: C{dict}
-
-    """
-
-    path = "/"
-
-    try:
-        prefix, id = id.split("|", 1)
-        path += "%s/" % (prefix,)
-    except ValueError:
-        # id has no prefix
-        pass
-
-    path += "%s/" % (pubSubConfiguration['host'],)
-    if id:
-        path += "%s/" % (id,)
-    return path
-
-def getPubSubXMPPURI(id, pubSubConfiguration):
-    return "xmpp:%s?pubsub;node=%s" % (pubSubConfiguration['service'],
-        getPubSubPath(id, pubSubConfiguration))
-
-def getPubSubHeartbeatURI(pubSubConfiguration):
-    return "xmpp:%s?pubsub;node=%s" % (pubSubConfiguration['service'],
-        getPubSubPath("", pubSubConfiguration))
-
-#
-# Notification Server service config
-#
-
-class NotificationOptions(Options):
-    optParameters = [[
-        "config", "f", DEFAULT_CONFIG_FILE, "Path to configuration file."
-    ]]
-
-    def __init__(self, *args, **kwargs):
-        super(NotificationOptions, self).__init__(*args, **kwargs)
-
-        self.overrides = {}
-
-    def _coerceOption(self, configDict, key, value):
-        """
-        Coerce the given C{val} to type of C{configDict[key]}
-        """
-        if key in configDict:
-            if isinstance(configDict[key], bool):
-                value = value == "True"
-
-            elif isinstance(configDict[key], (int, float, long)):
-                value = type(configDict[key])(value)
-
-            elif isinstance(configDict[key], (list, tuple)):
-                value = value.split(',')
-
-            elif isinstance(configDict[key], dict):
-                raise UsageError(
-                    "Dict options not supported on the command line"
-                )
-
-            elif value == 'None':
-                value = None
-
-        return value
-
-    def _setOverride(self, configDict, path, value, overrideDict):
-        """
-        Set the value at path in configDict
-        """
-        key = path[0]
-
-        if len(path) == 1:
-            overrideDict[key] = self._coerceOption(configDict, key, value)
-            return
-
-        if key in configDict:
-            if not isinstance(configDict[key], dict):
-                raise UsageError(
-                    "Found intermediate path element that is not a dictionary"
-                )
-
-            if key not in overrideDict:
-                overrideDict[key] = {}
-
-            self._setOverride(
-                configDict[key], path[1:],
-                value, overrideDict[key]
-            )
-
-
-    def opt_option(self, option):
-        """
-        Set an option to override a value in the config file. True, False, int,
-        and float options are supported, as well as comma seperated lists. Only
-        one option may be given for each --option flag, however multiple
-        --option flags may be specified.
-        """
-
-        if "=" in option:
-            path, value = option.split('=')
-            self._setOverride(
-                DEFAULT_CONFIG,
-                path.split('/'),
-                value,
-                self.overrides
-            )
-        else:
-            self.opt_option('%s=True' % (option,))
-
-    opt_o = opt_option
-
-    def postOptions(self):
-        config.load(self['config'])
-        config.updateDefaults(self.overrides)
-        self.parent['pidfile'] = None
-
-
-class NotificationServiceMaker(object):
-    implements(IPlugin, service.IServiceMaker)
-
-    tapname = "caldav_notifier"
-    description = "Notification Server"
-    options = NotificationOptions
-
-    def makeService(self, options):
-        try:
-            from setproctitle import setproctitle
-        except ImportError:
-            pass
-        else:
-            setproctitle("CalendarServer [Notification Gateway]")
-
-        #
-        # Configure Memcached Client Pool
-        #
-        memcachepool.installPools(
-            config.Memcached.Pools,
-            config.Memcached.MaxClients,
-        )
-
-        multiService = service.MultiService()
-
-        from calendarserver.tap.util import storeFromConfig, getDBPool
-        pool, txnFactory = getDBPool(config)
-        if pool is not None:
-            pool.setServiceParent(multiService)
-        store = storeFromConfig(config, txnFactory)
-
-        notifiers = []
-        for key, settings in config.Notifications.Services.iteritems():
-            if settings["Enabled"]:
-                notifier = namedClass(settings["Service"]).makeService(settings,
-                    store, config.ServerHostName)
-                notifier.setServiceParent(multiService)
-                notifiers.append(notifier)
-
-        internet.TCPServer(
-            config.Notifications.InternalNotificationPort,
-            InternalNotificationFactory(notifiers,
-                delaySeconds=config.Notifications.CoalesceSeconds),
-            interface=config.Notifications.BindAddress
-        ).setServiceParent(multiService)
-
-        return multiService
-
-
-class SimpleLineNotifierService(service.Service):
-
-    @classmethod
-    def makeService(cls, settings, store, serverHostName):
-        return cls(settings)
-
-    def __init__(self, settings):
-        self.notifier = SimpleLineNotifier(settings)
-        self.server = internet.TCPServer(settings["Port"],
-            SimpleLineNotificationFactory(self.notifier))
-
-    def enqueue(self, op, id):
-        self.notifier.enqueue(op, id)
-
-    def startService(self):
-        self.server.startService()
-
-    def stopService(self):
-        self.server.stopService()
-
-
-class XMPPNotifierService(service.Service):
-
-    @classmethod
-    def makeService(cls, settings, store, serverHostName):
-        return cls(settings)
-
-    def __init__(self, settings):
-        self.notifier = XMPPNotifier(settings)
-
-        if settings["Port"] == 5223: # use old SSL method
-            self.client = internet.SSLClient(settings["Host"], settings["Port"],
-                XMPPNotificationFactory(self.notifier, settings),
-                ClientContextFactory())
-        else:
-            # TLS and SASL
-            self.client = internet.TCPClient(settings["Host"], settings["Port"],
-                XMPPNotificationFactory(self.notifier, settings))
-
-    def enqueue(self, op, id):
-        self.notifier.enqueue(op, id)
-
-    def startService(self):
-        self.client.startService()
-
-    def stopService(self):
-        self.client.stopService()

Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/resource.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -75,10 +75,7 @@
 
 from twistedcaldav.icaldav import ICalDAVResource, ICalendarPrincipalResource
 from twistedcaldav.linkresource import LinkResource
-from twistedcaldav.notify import (
-    getPubSubConfiguration, getPubSubXMPPURI, getPubSubHeartbeatURI,
-    getPubSubAPSConfiguration,
-)
+from calendarserver.push.notifier import getPubSubAPSConfiguration
 from twistedcaldav.sharing import SharedCollectionMixin, SharedHomeMixin
 from twistedcaldav.util import normalizationLookup
 from twistedcaldav.vcard import Component as vComponent
@@ -2323,8 +2320,7 @@
 
         elif qname == (customxml.calendarserver_namespace, "push-transports"):
 
-            if (config.Notifications.Services.XMPPNotifier.Enabled or
-                config.Notifications.Services.ApplePushNotifier.Enabled):
+            if config.Notifications.Services.ApplePushNotifier.Enabled:
 
                 nodeName = (yield self._newStoreHome.nodeName())
                 if nodeName:
@@ -2354,78 +2350,18 @@
                                 )
                             )
 
-                        pubSubConfiguration = getPubSubConfiguration(config)
-                        if (pubSubConfiguration['enabled'] and
-                            pubSubConfiguration['xmpp-server']):
-                            children.append(
-                                customxml.PubSubTransportProperty(
-                                    customxml.PubSubXMPPServerProperty(
-                                        pubSubConfiguration['xmpp-server']
-                                    ),
-                                    customxml.PubSubXMPPURIProperty(
-                                        getPubSubXMPPURI(notifierID, pubSubConfiguration)
-                                    ),
-                                    type="XMPP",
-                                )
-                            )
 
                         returnValue(customxml.PubSubPushTransportsProperty(*children))
             returnValue(None)
 
         elif qname == (customxml.calendarserver_namespace, "pushkey"):
-            if (config.Notifications.Services.XMPPNotifier.Enabled or
-                config.Notifications.Services.AMPNotifier.Enabled or
+            if (config.Notifications.Services.AMPNotifier.Enabled or
                 config.Notifications.Services.ApplePushNotifier.Enabled):
                 nodeName = (yield self._newStoreHome.nodeName())
                 if nodeName:
                     returnValue(customxml.PubSubXMPPPushKeyProperty(nodeName))
             returnValue(None)
 
-        elif qname == (customxml.calendarserver_namespace, "xmpp-uri"):
-            if config.Notifications.Services.XMPPNotifier.Enabled:
-                nodeName = (yield self._newStoreHome.nodeName())
-                if nodeName:
-                    notifierID = self._newStoreHome.notifierID()
-                    if notifierID:
-                        pubSubConfiguration = getPubSubConfiguration(config)
-                        returnValue(customxml.PubSubXMPPURIProperty(
-                            getPubSubXMPPURI(notifierID, pubSubConfiguration)))
-
-            returnValue(None)
-
-        elif qname == (customxml.calendarserver_namespace, "xmpp-heartbeat-uri"):
-            if config.Notifications.Services.XMPPNotifier.Enabled:
-                # Look up node name not because we want to return it, but
-                # to see if XMPP server is actually responding.  If it comes
-                # back with an empty nodeName, don't advertise
-                # xmpp-heartbeat-uri
-                nodeName = (yield self._newStoreHome.nodeName())
-                if nodeName:
-                    pubSubConfiguration = getPubSubConfiguration(config)
-                    returnValue(
-                        customxml.PubSubHeartbeatProperty(
-                            customxml.PubSubHeartbeatURIProperty(
-                                getPubSubHeartbeatURI(pubSubConfiguration)
-                            ),
-                            customxml.PubSubHeartbeatMinutesProperty(
-                                str(pubSubConfiguration['heartrate'])
-                            )
-                        )
-                    )
-            returnValue(None)
-
-        elif qname == (customxml.calendarserver_namespace, "xmpp-server"):
-            if config.Notifications.Services.XMPPNotifier.Enabled:
-                # Look up node name not because we want to return it, but
-                # to see if XMPP server is actually responding.  If it comes
-                # back with an empty nodeName, don't advertise xmpp-server
-                nodeName = (yield self._newStoreHome.nodeName())
-                if nodeName:
-                    pubSubConfiguration = getPubSubConfiguration(config)
-                    returnValue(customxml.PubSubXMPPServerProperty(
-                        pubSubConfiguration['xmpp-server']))
-            returnValue(None)
-
         returnValue((yield super(CommonHomeResource, self).readProperty(property, request)))
 
 
@@ -2558,11 +2494,6 @@
             #caldavxml.DefaultAlarmVToDoDate.qname(),
 
         )
-        existing += (
-            (customxml.calendarserver_namespace, "xmpp-uri"),
-            (customxml.calendarserver_namespace, "xmpp-heartbeat-uri"),
-            (customxml.calendarserver_namespace, "xmpp-server"),
-        )
 
         if config.EnableManagedAttachments:
             existing += (

Modified: CalendarServer/trunk/twistedcaldav/scheduling/imip/delivery.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/delivery.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/delivery.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -19,25 +19,20 @@
 Handles the sending of scheduling messages via iMIP (mail gateway).
 """
 
-from twisted.python.failure import Failure
-from twisted.internet.defer import inlineCallbacks, returnValue
-
 from twext.python.log import Logger
-from twext.web2.dav.http import ErrorResponse
-
 from twext.web2 import responsecode
+from twext.web2.dav.http import ErrorResponse
 from twext.web2.http import HTTPError
-from twisted.web import client
-
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python.failure import Failure
 from twistedcaldav.caldavxml import caldav_namespace
-from twistedcaldav.config import config
-from twistedcaldav.util import AuthorizedHTTPGetter
+from twistedcaldav.directory.util import transactionFromRequest
 from twistedcaldav.scheduling.delivery import DeliveryService
+from twistedcaldav.scheduling.imip.outbound import IMIPInvitationWork
 from twistedcaldav.scheduling.itip import iTIPRequestStatus
-from twext.internet.gaiendpoint import GAIEndpoint
-from twext.internet.adaptendpoint import connect
 
 
+
 __all__ = [
     "ScheduleViaIMip",
 ]
@@ -105,9 +100,11 @@
 
                     fromAddr = str(self.scheduler.originator.cuaddr)
 
-                    log.debug("POSTing iMIP message to gateway...  To: '%s', From :'%s'\n%s" % (toAddr, fromAddr, caldata,))
-                    yield self.postToGateway(fromAddr, toAddr, caldata)
+                    txn = transactionFromRequest(self.scheduler.request, self.scheduler.request._newStoreTransaction.store)
+                    log.debug("Submitting iMIP message...  To: '%s', From :'%s'\n%s" % (toAddr, fromAddr, caldata,))
+                    yield txn.enqueue(IMIPInvitationWork, fromAddr=fromAddr, toAddr=toAddr, icalendarText=caldata)
 
+
                 except Exception, e:
                     # Generated failed response for this recipient
                     log.debug("iMIP request %s failed for recipient %s: %s" % (self, recipient, e))
@@ -125,26 +122,3 @@
             log.debug("iMIP request %s failed: %s" % (self, e))
             for recipient in self.recipients:
                 failForRecipient(recipient)
-
-
-    def postToGateway(self, fromAddr, toAddr, caldata, reactor=None):
-        if reactor is None:
-            from twisted.internet import reactor
-
-        mailGatewayServer = config.Scheduling['iMIP']['MailGatewayServer']
-        mailGatewayPort = config.Scheduling['iMIP']['MailGatewayPort']
-        url = "http://%s:%d/inbox" % (mailGatewayServer, mailGatewayPort)
-        headers = {
-            'Content-Type' : 'text/calendar',
-            'Originator' : fromAddr,
-            'Recipient' : toAddr,
-            config.Scheduling.iMIP.Header : config.Scheduling.iMIP.Password,
-        }
-        factory = client.HTTPClientFactory(url, method='POST', headers=headers,
-            postdata=caldata, agent="CalDAV server")
-
-        factory.noisy = False
-        factory.protocol = AuthorizedHTTPGetter
-        connect(GAIEndpoint(reactor, mailGatewayServer, mailGatewayPort),
-                factory)
-        return factory.deferred

Added: CalendarServer/trunk/twistedcaldav/scheduling/imip/inbound.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/inbound.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/inbound.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,632 @@
+##
+# Copyright (c) 2005-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Inbound IMIP mail handling for Calendar Server
+"""
+import datetime
+from calendarserver.tap.util import FakeRequest
+import email.utils
+from twext.enterprise.dal.record import fromTable
+from twext.enterprise.queue import WorkItem
+from twext.python.log import Logger, LoggingMixIn
+from twisted.application import service
+from twisted.internet import protocol, defer, ssl
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twisted.mail import pop3client, imap4
+from twisted.mail.smtp import messageid
+from twistedcaldav.config import config
+from twistedcaldav.ical import Property, Component
+from twistedcaldav.scheduling.imip.scheduler import IMIPScheduler
+from twistedcaldav.scheduling.imip.smtpsender import SMTPSender
+from twistedcaldav.scheduling.itip import iTIPRequestStatus
+from txdav.common.datastore.sql_tables import schema
+from twext.internet.gaiendpoint import GAIEndpoint
+
+
+log = Logger()
+
+#
+# Monkey patch imap4.log so it doesn't emit useless logging,
+# specifically, "Unhandled unsolicited response" nonsense.
+#
+class IMAPLogger(Logger):
+    def emit(self, level, message, *args, **kwargs):
+        if message.startswith("Unhandled unsolicited response:"):
+            return
+
+        Logger.emit(self, level, message, *args, **kwargs)
+
+imap4.log = IMAPLogger()
+
+
+""" SCHEMA:
+create table IMIP_REPLY_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  ORGANIZER                     varchar(255) not null,
+  ATTENDEE                      varchar(255) not null,
+  ICALENDAR_TEXT                text         not null
+);
+create table IMIP_POLLING_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP)
+);
+"""
+
+class IMIPReplyWork(WorkItem, fromTable(schema.IMIP_REPLY_WORK)):
+
+    @inlineCallbacks
+    def doWork(self):
+        rootResource = self.transaction._rootResource
+        calendar = Component.fromString(self.icalendarText)
+        yield injectMessage(self.transaction, rootResource, self.organizer, self.attendee,
+            calendar)
+
+class IMIPPollingWork(WorkItem, fromTable(schema.IMIP_POLLING_WORK)):
+
+    # FIXME: delete all other polling work items
+    # FIXME: purge all old tokens here
+
+    @inlineCallbacks
+    def doWork(self):
+        mailRetriever = self.transaction._mailRetriever
+        if mailRetriever is not None:
+            try:
+                yield mailRetriever.fetchMail()
+            except Exception, e:
+                log.error("Failed to fetch mail (%s)" % (e,))
+            finally:
+                yield mailRetriever.scheduleNextPoll()
+
+
+class MailRetriever(service.Service):
+
+    def __init__(self, store, directory, settings, reactor=None):
+        self.store = store
+        self.settings = settings
+        if reactor is None:
+            from twisted.internet import reactor
+        self.reactor = reactor
+
+        self.mailReceiver = MailReceiver(store, directory)
+        mailType = settings['Type']
+        if mailType.lower().startswith('pop'):
+            self.factory = POP3DownloadFactory
+        else:
+            self.factory = IMAP4DownloadFactory
+
+        contextFactory = None
+        if settings["UseSSL"]:
+            contextFactory = ssl.ClientContextFactory()
+        self.point = GAIEndpoint(self.reactor, settings.Server,
+            settings.Port, contextFactory=contextFactory)
+
+    def startService(self):
+        return self.scheduleNextPoll(seconds=0)
+
+
+    def fetchMail(self):
+        return self.point.connect(self.factory(self.settings, self.mailReceiver))
+
+    @inlineCallbacks
+    def scheduleNextPoll(self, seconds=None):
+        if seconds is None:
+            seconds = self.settings["PollingSeconds"]
+        txn = self.store.newTransaction()
+        notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
+        yield txn.enqueue(IMIPPollingWork, notBefore=notBefore)
+        yield txn.commit()
+
+
+
+class MailReceiver(object):
+
+    NO_TOKEN = 0
+    UNKNOWN_TOKEN = 1
+    MALFORMED_TO_ADDRESS = 2
+    NO_ORGANIZER_ADDRESS = 3
+    REPLY_FORWARDED_TO_ORGANIZER = 4
+    INJECTION_SUBMITTED = 5
+
+    # What about purge( ) and lowercase( )
+    def __init__(self, store, directory):
+        self.store = store
+        self.directory = directory
+
+    def checkDSN(self, message):
+        # returns (isdsn, action, icalendar attachment)
+
+        report = deliveryStatus = calBody = None
+
+        for part in message.walk():
+            contentType = part.get_content_type()
+            if contentType == "multipart/report":
+                report = part
+                continue
+            elif contentType == "message/delivery-status":
+                deliveryStatus = part
+                continue
+            elif contentType == "message/rfc822":
+                #original = part
+                continue
+            elif contentType == "text/calendar":
+                calBody = part.get_payload(decode=True)
+                continue
+
+        if report is not None and deliveryStatus is not None:
+            # we have what appears to be a dsn
+
+            lines = str(deliveryStatus).split("\n")
+            for line in lines:
+                lower = line.lower()
+                if lower.startswith("action:"):
+                    # found action:
+                    action = lower.split(' ')[1]
+                    break
+            else:
+                action = None
+
+            return True, action, calBody
+
+        else:
+            # not a dsn
+            return False, None, None
+
+
+    def _extractToken(self, text):
+        try:
+            pre, _ignore_post = text.split('@')
+            pre, token = pre.split('+')
+            return token
+        except ValueError:
+            return None
+
+
+    @inlineCallbacks
+    def processDSN(self, calBody, msgId):
+        calendar = Component.fromString(calBody)
+        # Extract the token (from organizer property)
+        organizer = calendar.getOrganizer()
+        token = self._extractToken(organizer)
+        if not token:
+            log.error("Mail gateway can't find token in DSN %s" % (msgId,))
+            return
+
+        txn = self.store.newTransaction()
+        result = (yield txn.imipLookupByToken(token))
+        yield txn.commit()
+        try:
+            # Note the results are returned as utf-8 encoded strings
+            organizer, attendee, icaluid = result[0]
+        except:
+            # This isn't a token we recognize
+            log.error("Mail gateway found a token (%s) but didn't "
+                           "recognize it in message %s"
+                           % (token, msgId))
+            returnValue(self.UNKNOWN_TOKEN)
+
+        calendar.removeAllButOneAttendee(attendee)
+        calendar.getOrganizerProperty().setValue(organizer)
+        for comp in calendar.subcomponents():
+            if comp.name() == "VEVENT":
+                comp.addProperty(Property("REQUEST-STATUS",
+                    ["5.1", "Service unavailable"]))
+                break
+        else:
+            # no VEVENT in the calendar body.
+            # TODO: what to do in this case?
+            pass
+
+        log.warn("Mail gateway processing DSN %s" % (msgId,))
+        txn = self.store.newTransaction()
+        yield txn.enqueue(IMIPReplyWork, organizer=organizer, attendee=attendee,
+            icalendarText=str(calendar))
+        yield txn.commit()
+        returnValue(self.INJECTION_SUBMITTED)
+
+
+    @inlineCallbacks
+    def processReply(self, msg):
+        # extract the token from the To header
+        _ignore_name, addr = email.utils.parseaddr(msg['To'])
+        if addr:
+            # addr looks like: server_address+token at example.com
+            token = self._extractToken(addr)
+            if not token:
+                log.error("Mail gateway didn't find a token in message "
+                               "%s (%s)" % (msg['Message-ID'], msg['To']))
+                returnValue(self.NO_TOKEN)
+        else:
+            log.error("Mail gateway couldn't parse To: address (%s) in "
+                           "message %s" % (msg['To'], msg['Message-ID']))
+            returnValue(self.MALFORMED_TO_ADDRESS)
+
+        txn = self.store.newTransaction()
+        result = (yield txn.imipLookupByToken(token))
+        yield txn.commit()
+        try:
+            # Note the results are returned as utf-8 encoded strings
+            organizer, attendee, icaluid = result[0]
+        except:
+            # This isn't a token we recognize
+            log.error("Mail gateway found a token (%s) but didn't "
+                           "recognize it in message %s"
+                           % (token, msg['Message-ID']))
+            returnValue(self.UNKNOWN_TOKEN)
+
+        for part in msg.walk():
+            if part.get_content_type() == "text/calendar":
+                calBody = part.get_payload(decode=True)
+                break
+        else:
+            # No icalendar attachment
+            log.warn("Mail gateway didn't find an icalendar attachment "
+                          "in message %s" % (msg['Message-ID'],))
+
+            toAddr = None
+            fromAddr = attendee[7:]
+            if organizer.startswith("mailto:"):
+                toAddr = organizer[7:]
+            elif organizer.startswith("urn:uuid:"):
+                guid = organizer[9:]
+                record = self.directory.recordWithGUID(guid)
+                if record and record.emailAddresses:
+                    toAddr = list(record.emailAddresses)[0]
+
+            if toAddr is None:
+                log.error("Don't have an email address for the organizer; "
+                               "ignoring reply.")
+                returnValue(self.NO_ORGANIZER_ADDRESS)
+
+            settings = config.Scheduling["iMIP"]["Sending"]
+            smtpSender = SMTPSender(settings.Username, settings.Password,
+                settings.UseSSL, settings.Server, settings.Port)
+
+            del msg["From"]
+            msg["From"] = fromAddr
+            del msg["Reply-To"]
+            msg["Reply-To"] = fromAddr
+            del msg["To"]
+            msg["To"] = toAddr
+            log.warn("Mail gateway forwarding reply back to organizer")
+            yield smtpSender.sendMessage(fromAddr, toAddr, messageid(), msg)
+            returnValue(self.REPLY_FORWARDED_TO_ORGANIZER)
+
+        # Process the imip attachment; inject to calendar server
+
+        log.debug(calBody)
+        calendar = Component.fromString(calBody)
+        event = calendar.mainComponent()
+
+        calendar.removeAllButOneAttendee(attendee)
+        organizerProperty = calendar.getOrganizerProperty()
+        if organizerProperty is None:
+            # ORGANIZER is required per rfc2446 section 3.2.3
+            log.warn("Mail gateway didn't find an ORGANIZER in REPLY %s"
+                          % (msg['Message-ID'],))
+            event.addProperty(Property("ORGANIZER", organizer))
+        else:
+            organizerProperty.setValue(organizer)
+
+        if not calendar.getAttendees():
+            # The attendee we're expecting isn't there, so add it back
+            # with a SCHEDULE-STATUS of SERVICE_UNAVAILABLE.
+            # The organizer will then see that the reply was not successful.
+            attendeeProp = Property("ATTENDEE", attendee,
+                params={
+                    "SCHEDULE-STATUS": iTIPRequestStatus.SERVICE_UNAVAILABLE,
+                }
+            )
+            event.addProperty(attendeeProp)
+
+            # TODO: We have talked about sending an email to the reply-to
+            # at this point, to let them know that their reply was missing
+            # the appropriate ATTENDEE.  This will require a new localizable
+            # email template for the message.
+
+        txn = self.store.newTransaction()
+        yield txn.enqueue(IMIPReplyWork, organizer=organizer, attendee=attendee,
+            icalendarText=str(calendar))
+        yield txn.commit()
+        returnValue(self.INJECTION_SUBMITTED)
+
+
+    # returns a deferred
+    def inbound(self, message):
+
+        try:
+            msg = email.message_from_string(message)
+
+            isDSN, action, calBody = self.checkDSN(msg)
+            if isDSN:
+                if action == 'failed' and calBody:
+                    # This is a DSN we can handle
+                    return self.processDSN(calBody, msg['Message-ID'])
+                else:
+                    # It's a DSN without enough to go on
+                    log.error("Mail gateway can't process DSN %s"
+                                   % (msg['Message-ID'],))
+                    return succeed(None)
+
+            log.info("Mail gateway received message %s from %s to %s" %
+                (msg['Message-ID'], msg['From'], msg['To']))
+
+            return self.processReply(msg)
+
+        except Exception, e:
+            # Don't let a failure of any kind stop us
+            log.error("Failed to process message: %s" % (e,))
+        return succeed(None)
+
+ at inlineCallbacks
+def injectMessage(txn, root, organizer, attendee, calendar):
+
+    request = FakeRequest(root, None, "/", transaction=txn)
+    resource = root.getChild("principals")
+    scheduler = IMIPScheduler(request, resource)
+    scheduler.originator = attendee
+    scheduler.recipients = [organizer,]
+    scheduler.calendar = calendar
+
+    try:
+        results = (yield scheduler.doScheduling())
+        log.info("Successfully injected iMIP response from %s to %s" %
+            (attendee, organizer))
+    except Exception, e:
+        log.error("Failed to inject iMIP response (%s)" % (e,))
+        raise
+
+    returnValue(results)
+
+
+    
+
+
+#
+# POP3
+#
+
+
+class POP3DownloadProtocol(pop3client.POP3Client, LoggingMixIn):
+    allowInsecureLogin = False
+
+    def serverGreeting(self, greeting):
+        self.log_debug("POP servergreeting")
+        pop3client.POP3Client.serverGreeting(self, greeting)
+        login = self.login(self.factory.settings["Username"],
+            self.factory.settings["Password"])
+        login.addCallback(self.cbLoggedIn)
+        login.addErrback(self.cbLoginFailed)
+
+
+    def cbLoginFailed(self, reason):
+        self.log_error("POP3 login failed for %s" %
+            (self.factory.settings["Username"],))
+        return self.quit()
+
+
+    def cbLoggedIn(self, result):
+        self.log_debug("POP loggedin")
+        return self.listSize().addCallback(self.cbGotMessageSizes)
+
+
+    def cbGotMessageSizes(self, sizes):
+        self.log_debug("POP gotmessagesizes")
+        downloads = []
+        for i in range(len(sizes)):
+            downloads.append(self.retrieve(i).addCallback(self.cbDownloaded, i))
+        return defer.DeferredList(downloads).addCallback(self.cbFinished)
+
+
+    def cbDownloaded(self, lines, id):
+        self.log_debug("POP downloaded message %d" % (id,))
+        self.factory.handleMessage("\r\n".join(lines))
+        self.log_debug("POP deleting message %d" % (id,))
+        self.delete(id)
+
+
+    def cbFinished(self, results):
+        self.log_debug("POP finished")
+        return self.quit()
+
+
+
+class POP3DownloadFactory(protocol.ClientFactory, LoggingMixIn):
+    protocol = POP3DownloadProtocol
+
+    def __init__(self, settings, mailReceiver):
+        self.mailReceiver = mailReceiver
+        self.noisy = False
+
+
+    def clientConnectionLost(self, connector, reason):
+        self.connector = connector
+        self.log_debug("POP factory connection lost")
+
+
+    def clientConnectionFailed(self, connector, reason):
+        self.connector = connector
+        self.log_info("POP factory connection failed")
+
+
+    def handleMessage(self, message):
+        self.log_debug("POP factory handle message")
+        self.log_debug(message)
+        return self.mailReceiver.inbound(message)
+
+
+
+#
+# IMAP4
+#
+
+
+class IMAP4DownloadProtocol(imap4.IMAP4Client, LoggingMixIn):
+
+    def serverGreeting(self, capabilities):
+        self.log_debug("IMAP servergreeting")
+        return self.authenticate(self.factory.settings["Password"]
+            ).addCallback(self.cbLoggedIn
+            ).addErrback(self.ebAuthenticateFailed)
+
+
+    def ebLogError(self, error):
+        self.log_error("IMAP Error: %s" % (error,))
+
+
+    def ebAuthenticateFailed(self, reason):
+        self.log_debug("IMAP authenticate failed for %s, trying login" %
+            (self.factory.settings["Username"],))
+        return self.login(self.factory.settings["Username"],
+            self.factory.settings["Password"]
+            ).addCallback(self.cbLoggedIn
+            ).addErrback(self.ebLoginFailed)
+
+
+    def ebLoginFailed(self, reason):
+        self.log_error("IMAP login failed for %s" %
+            (self.factory.settings["Username"],))
+        self.transport.loseConnection()
+
+
+    def cbLoggedIn(self, result):
+        self.log_debug("IMAP logged in [%s]" % (self.state,))
+        self.select("Inbox").addCallback(self.cbInboxSelected)
+
+
+    def cbInboxSelected(self, result):
+        self.log_debug("IMAP Inbox selected [%s]" % (self.state,))
+        allMessages = imap4.MessageSet(1, None)
+        self.fetchUID(allMessages, True).addCallback(self.cbGotUIDs)
+
+
+    def cbGotUIDs(self, results):
+        self.log_debug("IMAP got uids [%s]" % (self.state,))
+        self.messageUIDs = [result['UID'] for result in results.values()]
+        self.messageCount = len(self.messageUIDs)
+        self.log_debug("IMAP Inbox has %d messages" % (self.messageCount,))
+        if self.messageCount:
+            self.fetchNextMessage()
+        else:
+            # No messages; close it out
+            self.close().addCallback(self.cbClosed)
+
+
+    def fetchNextMessage(self):
+        self.log_debug("IMAP in fetchnextmessage [%s]" % (self.state,))
+        if self.messageUIDs:
+            nextUID = self.messageUIDs.pop(0)
+            messageListToFetch = imap4.MessageSet(nextUID)
+            self.log_debug("Downloading message %d of %d (%s)" %
+                (self.messageCount - len(self.messageUIDs), self.messageCount,
+                nextUID))
+            self.fetchMessage(messageListToFetch, True).addCallback(
+                self.cbGotMessage, messageListToFetch).addErrback(
+                    self.ebLogError)
+        else:
+            self.log_debug("Seeing if anything new has arrived")
+            # Go back and see if any more messages have come in
+            self.expunge().addCallback(self.cbInboxSelected)
+
+
+    def cbGotMessage(self, results, messageList):
+        self.log_debug("IMAP in cbGotMessage [%s]" % (self.state,))
+        try:
+            messageData = results.values()[0]['RFC822']
+        except IndexError:
+            # results will be empty unless the "twistedmail-imap-flags-anywhere"
+            # patch from http://twistedmatrix.com/trac/ticket/1105 is applied
+            self.log_error("Skipping empty results -- apply twisted patch!")
+            self.fetchNextMessage()
+            return
+
+        d = self.factory.handleMessage(messageData)
+        if isinstance(d, defer.Deferred):
+            d.addCallback(self.cbFlagDeleted, messageList)
+        else:
+            # No deferred returned, so no need for addCallback( )
+            self.cbFlagDeleted(None, messageList)
+
+
+    def cbFlagDeleted(self, results, messageList):
+        self.addFlags(messageList, ("\\Deleted",),
+            uid=True).addCallback(self.cbMessageDeleted, messageList)
+
+
+    def cbMessageDeleted(self, results, messageList):
+        self.log_debug("IMAP in cbMessageDeleted [%s]" % (self.state,))
+        self.log_debug("Deleted message")
+        self.fetchNextMessage()
+
+
+    def cbClosed(self, results):
+        self.log_debug("IMAP in cbClosed [%s]" % (self.state,))
+        self.log_debug("Mailbox closed")
+        self.logout().addCallback(
+            lambda _: self.transport.loseConnection())
+
+
+    def rawDataReceived(self, data):
+        self.log_debug("RAW RECEIVED: %s" % (data,))
+        imap4.IMAP4Client.rawDataReceived(self, data)
+
+
+    def lineReceived(self, line):
+        self.log_debug("RECEIVED: %s" % (line,))
+        imap4.IMAP4Client.lineReceived(self, line)
+
+
+    def sendLine(self, line):
+        self.log_debug("SENDING: %s" % (line,))
+        imap4.IMAP4Client.sendLine(self, line)
+
+
+
+class IMAP4DownloadFactory(protocol.ClientFactory, LoggingMixIn):
+    protocol = IMAP4DownloadProtocol
+
+    def __init__(self, settings, mailReceiver):
+        self.log_debug("Setting up IMAPFactory")
+
+        self.settings = settings
+        self.mailReceiver = mailReceiver
+        self.noisy = False
+
+
+    def buildProtocol(self, addr):
+        p = protocol.ClientFactory.buildProtocol(self, addr)
+        username = self.settings["Username"]
+        p.registerAuthenticator(imap4.CramMD5ClientAuthenticator(username))
+        p.registerAuthenticator(imap4.LOGINAuthenticator(username))
+        p.registerAuthenticator(imap4.PLAINAuthenticator(username))
+        return p
+
+
+    def handleMessage(self, message):
+        self.log_debug("IMAP factory handle message")
+        self.log_debug(message)
+        return self.mailReceiver.inbound(message)
+
+
+    def clientConnectionLost(self, connector, reason):
+        self.connector = connector
+        self.log_debug("IMAP factory connection lost")
+
+
+    def clientConnectionFailed(self, connector, reason):
+        self.connector = connector
+        self.log_warn("IMAP factory connection failed")

Modified: CalendarServer/trunk/twistedcaldav/scheduling/imip/mailgateway.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/mailgateway.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/mailgateway.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -1,4 +1,3 @@
-# -*- test-case-name: twistedcaldav.test.test_mail -*-
 ##
 # Copyright (c) 2005-2013 Apple Inc. All rights reserved.
 #
@@ -16,480 +15,21 @@
 ##
 
 """
-Mail Gateway for Calendar Server
+SQLite implementation of mail token database (deprecated).  This only exists
+now in order to migrate tokens from sqlite to the new store.
 """
 
-from __future__ import with_statement
-
-from cStringIO import StringIO
-
-from calendarserver.tap.util import getRootResource, directoryFromConfig
-
-from email.mime.image import MIMEImage
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-
-from pycalendar.datetime import PyCalendarDateTime
-from pycalendar.duration import PyCalendarDuration
-
-from twext.internet.adaptendpoint import connect
-from twext.internet.gaiendpoint import GAIEndpoint
-from twext.python.log import Logger, LoggingMixIn
-from twext.web2 import server
-from twext.web2.channel.http import HTTPFactory
-
-from twisted.application import internet, service
-from twisted.internet import protocol, defer, ssl, reactor as _reactor
-from twisted.internet.defer import succeed
-from twisted.mail import pop3client, imap4
-from twisted.mail.smtp import messageid, rfc822date, ESMTPSenderFactory
-from twisted.plugin import IPlugin
-from twisted.python.usage import Options, UsageError
-from twisted.web import client
-from twisted.web.microdom import Text as DOMText, Element as DOMElement
-from twisted.web.microdom import parseString
-from twisted.web.template import XMLString, TEMPLATE_NAMESPACE, Element, renderer, flattenString, tags
-
-from twistedcaldav import memcachepool
-from twistedcaldav.config import config
-from twistedcaldav.ical import Property, Component
-from twistedcaldav.localization import translationTo, _
-from twistedcaldav.scheduling.cuaddress import normalizeCUAddr
-from twistedcaldav.scheduling.imip.resource import IMIPInvitationInboxResource
-from twistedcaldav.scheduling.itip import iTIPRequestStatus
-from twistedcaldav.sql import AbstractSQLDatabase
-from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
-from twistedcaldav.util import AuthorizedHTTPGetter
-
-from zope.interface import implements
-
 import datetime
-import email.utils
 import os
-import urlparse
 import uuid
 
+from twext.python.log import Logger, LoggingMixIn
+from twistedcaldav.sql import AbstractSQLDatabase
+from twisted.internet.defer import inlineCallbacks
 
-__all__ = [
-    "MailGatewayServiceMaker",
-    "MailGatewayTokensDatabase",
-    "MailHandler",
-]
-
-
 log = Logger()
 
-#
-# Monkey patch imap4.log so it doesn't emit useless logging,
-# specifically, "Unhandled unsolicited response" nonsense.
-#
-class IMAPLogger(Logger):
-    def emit(self, level, message, *args, **kwargs):
-        if message.startswith("Unhandled unsolicited response:"):
-            return
 
-        Logger.emit(self, level, message, *args, **kwargs)
-
-imap4.log = IMAPLogger()
-
-#
-# Templates
-#
-
-plainCancelTemplate = u"""%(subject)s
-
-%(orgLabel)s: %(plainOrganizer)s
-%(dateLabel)s: %(dateInfo)s %(recurrenceInfo)s
-%(timeLabel)s: %(timeInfo)s %(durationInfo)s
-"""
-
-plainInviteTemplate = u"""%(subject)s
-
-%(orgLabel)s: %(plainOrganizer)s
-%(locLabel)s: %(location)s
-%(dateLabel)s: %(dateInfo)s %(recurrenceInfo)s
-%(timeLabel)s: %(timeInfo)s %(durationInfo)s
-%(descLabel)s: %(description)s
-%(urlLabel)s: %(url)s
-%(attLabel)s: %(plainAttendees)s
-"""
-
-
-htmlCancelTemplate = u"""<html>
-    <body><div>
-
-    <h1>%(subject)s</h1>
-    <p>
-    <h3>%(orgLabel)s:</h3> %(htmlOrganizer)s
-    </p>
-    <p>
-    <h3>%(dateLabel)s:</h3> %(dateInfo)s %(recurrenceInfo)s
-    </p>
-    <p>
-    <h3>%(timeLabel)s:</h3> %(timeInfo)s %(durationInfo)s
-    </p>
-    """.encode("utf-8")
-
-
-htmlInviteTemplate = u"""<html>
-    <body><div>
-    <p>%(inviteLabel)s</p>
-
-    <h1>%(summary)s</h1>
-    <p>
-    <h3>%(orgLabel)s:</h3> %(htmlOrganizer)s
-    </p>
-    <p>
-    <h3>%(locLabel)s:</h3> %(location)s
-    </p>
-    <p>
-    <h3>%(dateLabel)s:</h3> %(dateInfo)s %(recurrenceInfo)s
-    </p>
-    <p>
-    <h3>%(timeLabel)s:</h3> %(timeInfo)s %(durationInfo)s
-    </p>
-    <p>
-    <h3>%(descLabel)s:</h3> %(description)s
-    </p>
-    <p>
-    <h3>%(urlLabel)s:</h3> <a href="%(url)s">%(url)s</a>
-    </p>
-    <p>
-    <h3>%(attLabel)s:</h3> %(htmlAttendees)s
-    </p>
-    """.encode("utf-8")
-
-def _visit(document, node):
-    if isinstance(node, DOMText):
-        idx = node.parentNode.childNodes.index(node)
-        splitted = node.data.split("%(")
-        firstTextNode = document.createTextNode(splitted[0])
-        firstTextNode.parentNode = node.parentNode
-        replacements = [firstTextNode]
-        for moreText in splitted[1:]:
-            slotName, extra = moreText.split(')', 1)
-            extra = extra[1:]
-            slotElement = document.createElement('t:slot')
-            slotElement.setAttribute("name", slotName)
-            slotElement.parentNode = node.parentNode
-            textNode = document.createTextNode(extra)
-            textNode.parentNode = node.parentNode
-            replacements.append(slotElement)
-            replacements.append(textNode)
-        node.parentNode.childNodes[idx:idx + 1] = replacements
-
-    elif isinstance(node, DOMElement):
-        for attrName, attrVal in node.attributes.items():
-            if '%(' in attrVal:
-                del node.attributes[attrName]
-                elem = document.createElement('t:attr')
-                elem.setAttribute('name', attrName)
-                textNode = document.createTextNode(attrVal)
-                elem.appendChild(textNode)
-                node.appendChild(elem)
-
-
-
-def _walk(document, n):
-    _visit(document, n)
-    for subn in n.childNodes:
-        _walk(document, subn)
-
-
-
-def _fixup(data, rendererName):
-    document = parseString(data, beExtremelyLenient=True)
-    document.documentElement.setAttribute(
-        "xmlns:t", TEMPLATE_NAMESPACE
-    )
-    document.doctype = (
-        'html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" '
-        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
-    )
-    document.documentElement.setAttribute(
-        "t:render", rendererName
-    )
-    _walk(document, document)
-    result = document.toxml()
-    return result
-
-
-
-class StringFormatTemplateLoader(object):
-    """
-    Loader for twisted.web.template that converts a template with %()s slots.
-    """
-    def __init__(self, fileFactory, rendererName):
-        """
-        @param fileFactory: a 1-argument callable which returns a file-like
-            object that contains the %()s-format template.
-
-        @param rendererName: the name of the renderer.
-
-        @type rendererName: C{str}
-        """
-        self.fileFactory = fileFactory
-        self.rendererName = rendererName
-
-
-    def load(self):
-        html = _fixup(self.fileFactory().read(), self.rendererName)
-        return XMLString(html).load()
-
-
-
-def localizedLabels(language, canceled, inviteState):
-    """
-    Generate localized labels for an email in the given language.
-
-    @param language: a 2-letter language code
-
-    @type language: C{str}
-
-    @return: a 2-tuple of (subjectFormatString, labelDict), where the first is a
-        format string for use in the subject, and the latter is a dictionary
-        with labels suitable for filling out HTML and plain-text templates.  All
-        values are C{str}s.
-    """
-    with translationTo(language):
-        if canceled:
-            subjectFormatString = _("Event canceled: %(summary)s")
-        elif inviteState == "new":
-            subjectFormatString = _("Event invitation: %(summary)s")
-        elif inviteState == "update":
-            subjectFormatString = _("Event update: %(summary)s")
-        else:
-            subjectFormatString = _("Event reply: %(summary)s")
-
-        if canceled:
-            inviteLabel = _("Event Canceled")
-        else:
-            if inviteState == "new":
-                inviteLabel = _("Event Invitation")
-            elif inviteState == "update":
-                inviteLabel = _("Event Update")
-            else:
-                inviteLabel = _("Event Reply")
-
-        labels = dict(
-            dateLabel=_("Date"),
-            timeLabel=_("Time"),
-            durationLabel=_("Duration"),
-            recurrenceLabel=_("Occurs"),
-            descLabel=_("Description"),
-            urlLabel=_("URL"),
-            orgLabel=_("Organizer"),
-            attLabel=_("Attendees"),
-            locLabel=_("Location"),
-            inviteLabel=inviteLabel,
-        )
-
-        # The translations we get back from gettext are utf-8 encoded
-        # strings, so convert to unicode
-        for key in labels.keys():
-            if isinstance(labels[key], str):
-                labels[key] = labels[key].decode("utf-8")
-
-    return subjectFormatString.decode("utf-8"), labels
-
-
-
-class MailGatewayOptions(Options):
-    """
-    Mail gateway service config
-    """
-    optParameters = [[
-        "config", "f", DEFAULT_CONFIG_FILE, "Path to configuration file."
-    ]]
-
-    def __init__(self, *args, **kwargs):
-        super(MailGatewayOptions, self).__init__(*args, **kwargs)
-
-        self.overrides = {}
-
-
-    def _coerceOption(self, configDict, key, value):
-        """
-        Coerce the given C{val} to type of C{configDict[key]}
-        """
-        if key in configDict:
-            if isinstance(configDict[key], bool):
-                value = value == "True"
-
-            elif isinstance(configDict[key], (int, float, long)):
-                value = type(configDict[key])(value)
-
-            elif isinstance(configDict[key], (list, tuple)):
-                value = value.split(',')
-
-            elif isinstance(configDict[key], dict):
-                raise UsageError(
-                    "Dict options not supported on the command line"
-                )
-
-            elif value == 'None':
-                value = None
-
-        return value
-
-
-    def _setOverride(self, configDict, path, value, overrideDict):
-        """
-        Set the value at path in configDict
-        """
-        key = path[0]
-
-        if len(path) == 1:
-            overrideDict[key] = self._coerceOption(configDict, key, value)
-            return
-
-        if key in configDict:
-            if not isinstance(configDict[key], dict):
-                raise UsageError(
-                    "Found intermediate path element that is not a dictionary"
-                )
-
-            if key not in overrideDict:
-                overrideDict[key] = {}
-
-            self._setOverride(
-                configDict[key], path[1:],
-                value, overrideDict[key]
-            )
-
-
-    def opt_option(self, option):
-        """
-        Set an option to override a value in the config file. True, False, int,
-        and float options are supported, as well as comma separated lists. Only
-        one option may be given for each --option flag, however multiple
-        --option flags may be specified.
-        """
-
-        if "=" in option:
-            path, value = option.split('=')
-            self._setOverride(
-                DEFAULT_CONFIG,
-                path.split('/'),
-                value,
-                self.overrides
-            )
-        else:
-            self.opt_option('%s=True' % (option,))
-
-    opt_o = opt_option
-
-    def postOptions(self):
-        config.load(self['config'])
-        config.updateDefaults(self.overrides)
-        self.parent['pidfile'] = None
-
-
-
-def injectionSettingsFromURL(url, config):
-    """
-    Given a url returned from server podding info (or None if not podding),
-    generate the url that should be used to inject an iMIP reply.  If the
-    url is None, then compute the url from config.
-    """
-    path = "inbox"
-    if url is None:
-        # Didn't get url from server podding configuration, so use caldavd.plist
-        if config.Scheduling.iMIP.MailGatewayServer == "localhost":
-            hostname = "localhost"
-        else:
-            hostname = config.ServerHostName
-        if config.EnableSSL:
-            useSSL = True
-            port = config.SSLPort
-        else:
-            useSSL = False
-            port = config.HTTPPort
-        scheme = "https:" if useSSL else "http:"
-        url = "%s//%s:%d/%s/" % (scheme, hostname, port, path)
-    else:
-        url = "%s/%s/" % (url.rstrip("/"), path)
-    return url
-
-
-
-def injectMessage(url, organizer, attendee, calendar, msgId, reactor=None):
-
-    if reactor is None:
-        reactor = _reactor
-
-    headers = {
-        'Content-Type' : 'text/calendar',
-        'Originator' : attendee,
-        'Recipient' : organizer,
-        config.Scheduling.iMIP.Header : config.Scheduling.iMIP.Password,
-    }
-
-    data = str(calendar)
-    url = injectionSettingsFromURL(url, config)
-    parsed = urlparse.urlparse(url)
-
-    log.debug("Injecting to %s: %s %s" % (url, str(headers), data))
-
-    factory = client.HTTPClientFactory(url, method='POST', headers=headers,
-        postdata=data, agent="iMIP gateway")
-
-    factory.noisy = False
-    factory.protocol = AuthorizedHTTPGetter
-
-    if parsed.scheme == "https":
-        connect(GAIEndpoint(reactor, parsed.hostname, parsed.port,
-                            ssl.ClientContextFactory()),
-                factory)
-    else:
-        connect(GAIEndpoint(reactor, parsed.hostname, parsed.port), factory)
-
-
-    def _success(result, msgId):
-        log.info("Mail gateway successfully injected message %s" % (msgId,))
-
-
-    def _failure(failure, msgId):
-        log.err("Mail gateway failed to inject message %s (Reason: %s)" %
-            (msgId, failure.getErrorMessage()))
-        log.debug("Failed calendar body: %s" % (str(calendar),))
-
-    factory.deferred.addCallback(_success, msgId).addErrback(_failure, msgId)
-    return factory.deferred
-
-
-
-def serverForOrganizer(directory, organizer):
-    """
-    Return the URL for the server hosting the organizer, or None if podding
-    is not enabled or organizer is hosted locally.
-    Raises ServerNotFound if we can't find the record for the organizer.
-    @param directory: service to look for organizer in
-    @type directory: L{DirectoryService}
-    @param organizer: CUA of organizer
-    @type organizer: C{str}
-    @return: string URL
-    """
-    record = directory.recordWithCalendarUserAddress(organizer)
-    if record is None:
-        log.warn("Can't find server for %s" % (organizer,))
-        raise ServerNotFound()
-
-    srvr = record.server()  # None means hosted locally
-    if srvr is None:
-        return None
-    else:
-        return srvr.uri
-
-
-
-class ServerNotFound(Exception):
-    """
-    Can't determine which server is hosting a given user
-    """
-
-
-
 class MailGatewayTokensDatabase(AbstractSQLDatabase, LoggingMixIn):
     """
     A database to maintain "plus-address" tokens for IMIP requests.
@@ -560,6 +100,15 @@
         else:
             return None
 
+    def getAllTokens(self):
+        results = list(
+            self._db_execute(
+                """
+                select TOKEN, ORGANIZER, ATTENDEE, ICALUID from TOKENS
+                """
+            )
+        )
+        return results
 
     def deleteToken(self, token):
         self._db_execute(
@@ -665,1220 +214,20 @@
         pass
 
 
-
-#
-# Service
-#
-
-class MailGatewayService(service.MultiService):
-
-    def startService(self):
-        """
-        Purge old database tokens -- doing this in startService so that
-        it happens after we've shed privileges
-        """
-        service.MultiService.startService(self)
-        mailer = getattr(self, "mailer", None)
-        if mailer is not None:
-            mailer.purge()
-            mailer.lowercase()
-
-
-
-class MailGatewayServiceMaker(LoggingMixIn):
-    implements(IPlugin, service.IServiceMaker)
-
-    tapname = "caldav_mailgateway"
-    description = "Mail Gateway"
-    options = MailGatewayOptions
-
-    def makeService(self, options):
-        try:
-            from setproctitle import setproctitle
-        except ImportError:
-            pass
-        else:
-            setproctitle("CalendarServer [Mail Gateway]")
-
-        memcachepool.installPools(
-            config.Memcached.Pools,
-            config.Memcached.MaxClients,
-        )
-
-        mailGatewayService = MailGatewayService()
-
-        settings = config.Scheduling['iMIP']
-        if settings['Enabled']:
-            mailer = MailHandler()
-
-            mailType = settings['Receiving']['Type']
-            if mailType.lower().startswith('pop'):
-                self.log_info("Starting Mail Gateway Service: POP3")
-                client = POP3Service(settings['Receiving'], mailer)
-            elif mailType.lower().startswith('imap'):
-                self.log_info("Starting Mail Gateway Service: IMAP4")
-                client = IMAP4Service(settings['Receiving'], mailer)
-            else:
-                # TODO: raise error?
-                self.log_error("Invalid iMIP type in configuration: %s" %
-                    (mailType,))
-                return mailGatewayService
-
-            client.setServiceParent(mailGatewayService)
-
-            # Set up /inbox -- server POSTs to it to send out iMIP invites
-            IScheduleService(settings, mailer).setServiceParent(
-                mailGatewayService
-            )
-
-        else:
-            mailer = None
-            self.log_info("Mail Gateway Service not enabled")
-
-        mailGatewayService.mailer = mailer
-        return mailGatewayService
-
-
-
-class IScheduleService(service.MultiService, LoggingMixIn):
+ at inlineCallbacks
+def migrateTokensToStore(path, store):
     """
-    ISchedule Inbox
-    """
+    Copy all the tokens from the sqlite db into the new store.
 
-    def __init__(self, settings, mailer):
-        service.MultiService.__init__(self)
-        self.settings = settings
-        self.mailer = mailer
+    @param path: Filesystem path to directory containing the sqlite db file.
+    @type path: C{str}
 
-        # Disable since we're only interested in /principals (for auth)
-        config.EnableCalDAV = False
-        config.EnableCardDAV = False
-
-        rootResource = getRootResource(
-            config,
-            "IGNORED", # no need for a store - no /calendars nor /addressbooks
-            resources=[
-                ("inbox", IMIPInvitationInboxResource, (mailer,), ("digest",)),
-            ]
-        )
-
-        self.factory = HTTPFactory(server.Site(rootResource))
-        self.server = internet.TCPServer(settings['MailGatewayPort'],
-            self.factory)
-        self.server.setServiceParent(self)
-
-
-
-class MailHandler(LoggingMixIn):
-
-    def __init__(self, dataRoot=None, directory=None):
-        if dataRoot is None:
-            dataRoot = config.DataRoot
-        if directory is None:
-            directory = directoryFromConfig(config)
-        self.db = MailGatewayTokensDatabase(dataRoot)
-        self.days = config.Scheduling['iMIP']['InvitationDaysToLive']
-        self.directory = directory
-
-
-    def purge(self):
-        """
-        Purge old database tokens
-        """
-        self.db.purgeOldTokens(datetime.date.today() -
-            datetime.timedelta(days=self.days))
-
-
-    def lowercase(self):
-        """
-        Convert all mailto: to lowercase
-        """
-        self.db.lowercase()
-
-
-    def checkDSN(self, message):
-        # returns (isDSN, Action, icalendar attachment)
-
-        report = deliveryStatus = calBody = None
-
-        for part in message.walk():
-            content_type = part.get_content_type()
-            if content_type == "multipart/report":
-                report = part
-                continue
-            elif content_type == "message/delivery-status":
-                deliveryStatus = part
-                continue
-            elif content_type == "message/rfc822":
-                #original = part
-                continue
-            elif content_type == "text/calendar":
-                calBody = part.get_payload(decode=True)
-                continue
-
-        if report is not None and deliveryStatus is not None:
-            # we have what appears to be a DSN
-
-            lines = str(deliveryStatus).split("\n")
-            for line in lines:
-                lower = line.lower()
-                if lower.startswith("action:"):
-                    # found Action:
-                    action = lower.split(' ')[1]
-                    break
-            else:
-                action = None
-
-            return True, action, calBody
-
-        else:
-            # Not a DSN
-            return False, None, None
-
-
-    def _extractToken(self, text):
-        try:
-            pre, _ignore_post = text.split('@')
-            pre, token = pre.split('+')
-            return token
-        except ValueError:
-            return None
-
-
-    def processDSN(self, calBody, msgId, fn):
-        calendar = Component.fromString(calBody)
-        # Extract the token (from organizer property)
-        organizer = calendar.getOrganizer()
-        token = self._extractToken(organizer)
-        if not token:
-            self.log_error("Mail gateway can't find token in DSN %s" % (msgId,))
-            return
-
-        result = self.db.lookupByToken(token)
-        if result is None:
-            # This isn't a token we recognize
-            self.log_error("Mail gateway found a token (%s) but didn't "
-                           "recognize it in DSN %s" % (token, msgId))
-            return
-
-        organizer, attendee, icaluid = result
-        organizer = str(organizer)
-        attendee = str(attendee)
-        icaluid = str(icaluid)
-        calendar.removeAllButOneAttendee(attendee)
-        calendar.getOrganizerProperty().setValue(organizer)
-        for comp in calendar.subcomponents():
-            if comp.name() == "VEVENT":
-                comp.addProperty(Property("REQUEST-STATUS",
-                    ["5.1", "Service unavailable"]))
-                break
-        else:
-            # no VEVENT in the calendar body.
-            # TODO: what to do in this case?
-            pass
-
-        try:
-            hostname = serverForOrganizer(self.directory, organizer)
-        except ServerNotFound:
-            # We can't determine which server hosts the organizer
-            self.log_error("Unable to determine which server hosts organizer %s"
-                % (organizer,))
-            return succeed(None)
-
-        self.log_warn("Mail gateway processing DSN %s to server %s" % (msgId, hostname))
-        return fn(hostname, organizer, attendee, calendar, msgId)
-
-
-    def processReply(self, msg, injectFunction, testMode=False):
-        # extract the token from the To header
-        _ignore_name, addr = email.utils.parseaddr(msg['To'])
-        if addr:
-            # addr looks like: server_address+token at example.com
-            token = self._extractToken(addr)
-            if not token:
-                self.log_error("Mail gateway didn't find a token in message "
-                               "%s (%s)" % (msg['Message-ID'], msg['To']))
-                return
-        else:
-            self.log_error("Mail gateway couldn't parse To: address (%s) in "
-                           "message %s" % (msg['To'], msg['Message-ID']))
-            return
-
-        result = self.db.lookupByToken(token)
-        if result is None:
-            # This isn't a token we recognize
-            self.log_error("Mail gateway found a token (%s) but didn't "
-                           "recognize it in message %s"
-                           % (token, msg['Message-ID']))
-            return
-
-        organizer, attendee, icaluid = result
-        organizer = str(organizer)
-        attendee = str(attendee)
-        icaluid = str(icaluid)
-
-        for part in msg.walk():
-            if part.get_content_type() == "text/calendar":
-                calBody = part.get_payload(decode=True)
-                break
-        else:
-            # No icalendar attachment
-            self.log_warn("Mail gateway didn't find an icalendar attachment "
-                          "in message %s" % (msg['Message-ID'],))
-
-            toAddr = None
-            fromAddr = attendee[7:]
-
-            if organizer.startswith("mailto:"):
-                toAddr = organizer[7:]
-            elif organizer.startswith("urn:uuid:"):
-                guid = organizer[9:]
-                record = self.directory.recordWithGUID(guid)
-                if record and record.emailAddresses:
-                    toAddr = list(record.emailAddresses)[0]
-
-            if toAddr is None:
-                self.log_error("Don't have an email address for the organizer; "
-                               "ignoring reply.")
-                return
-
-            if testMode:
-                return (toAddr, fromAddr)
-
-            settings = config.Scheduling["iMIP"]["Sending"]
-            if settings["UseSSL"]:
-                contextFactory = ssl.ClientContextFactory()
-            else:
-                contextFactory = None
-
-            deferred = defer.Deferred()
-            del msg["From"]
-            msg["From"] = fromAddr
-            del msg["Reply-To"]
-            msg["Reply-To"] = fromAddr
-            del msg["To"]
-            msg["To"] = toAddr
-            factory = ESMTPSenderFactory(
-                settings["Username"], settings["Password"],
-                fromAddr, toAddr,
-                # per http://trac.calendarserver.org/ticket/416 ...
-                StringIO(msg.as_string().replace("\r\n", "\n")),
-                deferred,
-                contextFactory=contextFactory,
-                requireAuthentication=False,
-                requireTransportSecurity=settings["UseSSL"],
-            )
-
-            self.log_warn("Mail gateway forwarding reply back to organizer")
-            connect(GAIEndpoint(_reactor, settings["Server"], settings["Port"]),
-                    factory)
-            return deferred
-
-        # Process the imip attachment; inject to calendar server
-
-        self.log_debug(calBody)
-        calendar = Component.fromString(calBody)
-        event = calendar.mainComponent()
-
-        calendar.removeAllButOneAttendee(attendee)
-        organizerProperty = calendar.getOrganizerProperty()
-        if organizerProperty is None:
-            # ORGANIZER is required per rfc2446 section 3.2.3
-            self.log_warn("Mail gateway didn't find an ORGANIZER in REPLY %s"
-                          % (msg['Message-ID'],))
-            event.addProperty(Property("ORGANIZER", organizer))
-        else:
-            organizerProperty.setValue(organizer)
-
-        if not calendar.getAttendees():
-            # The attendee we're expecting isn't there, so add it back
-            # with a SCHEDULE-STATUS of SERVICE_UNAVAILABLE.
-            # The organizer will then see that the reply was not successful.
-            attendeeProp = Property("ATTENDEE", attendee,
-                params={
-                    "SCHEDULE-STATUS": iTIPRequestStatus.SERVICE_UNAVAILABLE,
-                }
-            )
-            event.addProperty(attendeeProp)
-
-            # TODO: We have talked about sending an email to the reply-to
-            # at this point, to let them know that their reply was missing
-            # the appropriate ATTENDEE.  This will require a new localizable
-            # email template for the message.
-
-        try:
-            hostname = serverForOrganizer(self.directory, organizer)
-        except ServerNotFound:
-            # We can't determine which server hosts the organizer
-            self.log_error("Unable to determine which server hosts organizer %s"
-                % (organizer,))
-            return succeed(None)
-
-        return injectFunction(hostname, organizer, attendee, calendar,
-            msg['Message-ID'])
-
-
-    def inbound(self, message, fn=injectMessage):
-        try:
-            msg = email.message_from_string(message)
-
-            isDSN, action, calBody = self.checkDSN(msg)
-            if isDSN:
-                if action == 'failed' and calBody:
-                    # This is a DSN we can handle
-                    return self.processDSN(calBody, msg['Message-ID'], fn)
-                else:
-                    # It's a DSN without enough to go on
-                    self.log_error("Mail gateway can't process DSN %s"
-                                   % (msg['Message-ID'],))
-                    return
-
-            self.log_info("Mail gateway received message %s from %s to %s" %
-                (msg['Message-ID'], msg['From'], msg['To']))
-
-            return self.processReply(msg, fn)
-
-        except Exception, e:
-            # Don't let a failure of any kind stop us
-            self.log_error("Failed to process message: %s" % (e,))
-
-
-    def outbound(self, originator, recipient, calendar, language='en',
-                 send=True, onlyAfter=None):
-        # create token, send email
-
-        settings = config.Scheduling['iMIP']['Sending']
-
-        if onlyAfter is None:
-            duration = PyCalendarDuration(days=settings.SuppressionDays)
-            onlyAfter = PyCalendarDateTime.getNowUTC() - duration
-
-        component = calendar.masterComponent()
-        if component is None:
-            component = calendar.mainComponent(True)
-        icaluid = component.propertyValue("UID")
-        method = calendar.propertyValue("METHOD")
-
-        # Clean up the attendee list which is purely used within the human
-        # readable email message (not modifying the calendar body)
-        attendees = []
-        for attendeeProp in calendar.getAllAttendeeProperties():
-            cutype = attendeeProp.parameterValue("CUTYPE", "INDIVIDUAL")
-            if cutype == "INDIVIDUAL":
-                cn = attendeeProp.parameterValue("CN", None)
-                if cn is not None:
-                    cn = cn.decode("utf-8")
-                cuaddr = normalizeCUAddr(attendeeProp.value())
-                if cuaddr.startswith("mailto:"):
-                    mailto = cuaddr[7:]
-                    if not cn:
-                        cn = mailto
-                else:
-                    emailAddress = attendeeProp.parameterValue("EMAIL", None)
-                    if emailAddress:
-                        mailto = emailAddress
-                    else:
-                        mailto = None
-
-                if cn or mailto:
-                    attendees.append((cn, mailto))
-
-        toAddr = recipient
-        if not recipient.lower().startswith("mailto:"):
-            raise ValueError("ATTENDEE address '%s' must be mailto: for iMIP "
-                             "operation." % (recipient,))
-        recipient = recipient[7:]
-
-        if method != "REPLY":
-            # Invites and cancellations:
-
-            # Reuse or generate a token based on originator, toAddr, and
-            # event uid
-            token = self.db.getToken(originator, toAddr.lower(), icaluid)
-            if token is None:
-
-                # Because in the past the originator was sometimes in mailto:
-                # form, lookup an existing token by mailto: as well
-                organizerProperty = calendar.getOrganizerProperty()
-                organizerEmailAddress = organizerProperty.parameterValue("EMAIL", None)
-                if organizerEmailAddress is not None:
-                    token = self.db.getToken("mailto:%s" % (organizerEmailAddress.lower(),), toAddr.lower(), icaluid)
-
-            if token is None:
-                token = self.db.createToken(originator, toAddr.lower(), icaluid)
-                self.log_debug("Mail gateway created token %s for %s "
-                               "(originator), %s (recipient) and %s (icaluid)"
-                               % (token, originator, toAddr, icaluid))
-                inviteState = "new"
-
-            else:
-                self.log_debug("Mail gateway reusing token %s for %s "
-                               "(originator), %s (recipient) and %s (icaluid)"
-                               % (token, originator, toAddr, icaluid))
-                inviteState = "update"
-
-            fullServerAddress = settings['Address']
-            _ignore_name, serverAddress = email.utils.parseaddr(fullServerAddress)
-            pre, post = serverAddress.split('@')
-            addressWithToken = "%s+%s@%s" % (pre, token, post)
-
-            organizerProperty = calendar.getOrganizerProperty()
-            organizerEmailAddress = organizerProperty.parameterValue("EMAIL",
-                                                                     None)
-            organizerValue = organizerProperty.value()
-            organizerProperty.setValue("mailto:%s" % (addressWithToken,))
-
-            # If the organizer is also an attendee, update that attendee value
-            # to match
-            organizerAttendeeProperty = calendar.getAttendeeProperty(
-                [organizerValue])
-            if organizerAttendeeProperty is not None:
-                organizerAttendeeProperty.setValue("mailto:%s" %
-                                                   (addressWithToken,))
-
-            # The email's From will include the originator's real name email
-            # address if available.  Otherwise it will be the server's email
-            # address (without # + addressing)
-            if organizerEmailAddress:
-                orgEmail = fromAddr = organizerEmailAddress
-            else:
-                fromAddr = serverAddress
-                orgEmail = None
-            cn = calendar.getOrganizerProperty().parameterValue('CN', None)
-            if cn is None:
-                cn = u'Calendar Server'
-                orgCN = orgEmail
-            else:
-                orgCN = cn = cn.decode("utf-8")
-
-            # a unicode cn (rather than an encode string value) means the
-            # from address will get properly encoded per rfc2047 within the
-            # MIMEMultipart in generateEmail
-            formattedFrom = "%s <%s>" % (cn, fromAddr)
-
-            # Reply-to address will be the server+token address
-
-        else: # REPLY
-            inviteState = "reply"
-
-            # Look up the attendee property corresponding to the originator
-            # of this reply
-            originatorAttendeeProperty = calendar.getAttendeeProperty(
-                [originator])
-            formattedFrom = fromAddr = originator = ""
-            if originatorAttendeeProperty:
-                originatorAttendeeEmailAddress = (
-                    originatorAttendeeProperty.parameterValue("EMAIL", None)
-                )
-                if originatorAttendeeEmailAddress:
-                    formattedFrom = fromAddr = originator = (
-                        originatorAttendeeEmailAddress
-                    )
-
-            organizerMailto = str(calendar.getOrganizer())
-            if not organizerMailto.lower().startswith("mailto:"):
-                raise ValueError("ORGANIZER address '%s' must be mailto: "
-                                 "for REPLY." % (organizerMailto,))
-            orgEmail = organizerMailto[7:]
-
-            orgCN = calendar.getOrganizerProperty().parameterValue('CN', None)
-            addressWithToken = formattedFrom
-
-        # At the point we've created the token in the db, which we always
-        # want to do, but if this message is for an event completely in
-        # the past we don't want to actually send an email.
-        if not calendar.hasInstancesAfter(onlyAfter):
-            self.log_debug("Skipping IMIP message for old event")
-            return succeed(True)
-
-        # Now prevent any "internal" CUAs from being exposed by converting
-        # to mailto: if we have one
-        for attendeeProp in calendar.getAllAttendeeProperties():
-            cutype = attendeeProp.parameterValue('CUTYPE', None)
-            if cutype == "INDIVIDUAL":
-                cuaddr = normalizeCUAddr(attendeeProp.value())
-                if not cuaddr.startswith("mailto:"):
-                    emailAddress = attendeeProp.parameterValue("EMAIL", None)
-                    if emailAddress:
-                        attendeeProp.setValue("mailto:%s" % (emailAddress,))
-
-        msgId, message = self.generateEmail(inviteState, calendar, orgEmail,
-            orgCN, attendees, formattedFrom, addressWithToken, recipient,
-            language=language)
-
-        if send:
-            self.log_debug("Sending: %s" % (message,))
-            def _success(result, msgId, fromAddr, toAddr):
-                self.log_info("Mail gateway sent message %s from %s to %s" %
-                    (msgId, fromAddr, toAddr))
-                return True
-
-            def _failure(failure, msgId, fromAddr, toAddr):
-                self.log_error("Mail gateway failed to send message %s from %s "
-                               "to %s (Reason: %s)" %
-                               (msgId, fromAddr, toAddr,
-                                failure.getErrorMessage()))
-                return False
-
-            deferred = defer.Deferred()
-
-            if settings["UseSSL"]:
-                contextFactory = ssl.ClientContextFactory()
-            else:
-                contextFactory = None
-
-            factory = ESMTPSenderFactory(
-                settings['Username'], settings['Password'],
-                fromAddr, toAddr, StringIO(str(message)), deferred,
-                contextFactory=contextFactory,
-                requireAuthentication=False,
-                requireTransportSecurity=settings["UseSSL"])
-
-            connect(GAIEndpoint(_reactor, settings["Server"], settings["Port"]),
-                    factory)
-            deferred.addCallback(_success, msgId, fromAddr, toAddr)
-            deferred.addErrback(_failure, msgId, fromAddr, toAddr)
-            return deferred
-        else:
-            return succeed((inviteState, calendar, orgEmail, orgCN, attendees,
-                formattedFrom, recipient, addressWithToken))
-
-
-    def getIconPath(self, details, canceled, language='en'):
-        iconDir = config.Scheduling.iMIP.MailIconsDirectory.rstrip("/")
-
-        if canceled:
-            iconName = "canceled.png"
-            iconPath = os.path.join(iconDir, iconName)
-            if os.path.exists(iconPath):
-                return iconPath
-            else:
-                return None
-
-        else:
-            month = int(details['month'])
-            day = int(details['day'])
-            with translationTo(language) as trans:
-                monthName = trans.monthAbbreviation(month)
-            iconName = "%02d.png" % (day,)
-            iconPath = os.path.join(iconDir, monthName.encode("utf-8"), iconName)
-            if not os.path.exists(iconPath):
-                # Try the generic (numeric) version
-                iconPath = os.path.join(iconDir, "%02d" % (month,), iconName)
-                if not os.path.exists(iconPath):
-                    return None
-            return iconPath
-
-
-    def generateEmail(self, inviteState, calendar, orgEmail, orgCN,
-                      attendees, fromAddress, replyToAddress, toAddress,
-                      language='en'):
-        """
-        Generate MIME text containing an iMIP invitation, cancellation, update
-        or reply.
-
-        @param inviteState: 'new', 'update', or 'reply'.
-
-        @type inviteState: C{str}
-
-        @param calendar: the iCalendar component to attach to the email.
-
-        @type calendar: L{twistedcaldav.ical.Component}
-
-        @param orgEmail: The email for the organizer, in C{localhost at domain}
-            format, or C{None} if the organizer has no email address.
-
-        @type orgEmail: C{str} or C{NoneType}
-
-        @param orgCN: Common name / display name for the organizer.
-
-        @type orgCN: C{unicode}
-
-        @param attendees: A C{list} of 2-C{tuple}s of (common name, email
-            address) similar to (orgEmail, orgCN).
-
-        @param fromAddress: the address to use in the C{From:} header of the
-            email.
-
-        @type fromAddress: C{str}
-
-        @param replyToAddress: the address to use in the C{Reply-To} header.
-
-        @type replyToAddress: C{str}
-
-        @param toAddress: the address to use in the C{To} header.
-
-        @type toAddress: C{str}
-
-        @param language: a 2-letter language code describing the target
-            language that the email should be generated in.
-
-        @type language: C{str}
-
-        @return: a 2-tuple of C{str}s: (message ID, message text).  The message
-            ID is the value of the C{Message-ID} header, and the message text is
-            the full MIME message, ready for transport over SMTP.
-        """
-
-        details = self.getEventDetails(calendar, language=language)
-        canceled = (calendar.propertyValue("METHOD") == "CANCEL")
-        iconPath = self.getIconPath(details, canceled, language=language)
-
-        subjectFormat, labels = localizedLabels(language, canceled, inviteState)
-        details.update(labels)
-
-        details['subject'] = subjectFormat % {'summary' : details['summary']}
-        details['iconName'] = iconName = "calicon.png"
-
-        plainText = self.renderPlainText(details, (orgCN, orgEmail),
-                                         attendees, canceled)
-
-        [addIcon, htmlText] = self.renderHTML(details, (orgCN, orgEmail),
-                                              attendees, canceled)
-
-        msg = MIMEMultipart()
-        msg["From"] = fromAddress
-        msg["Subject"] = details['subject']
-        msg["Reply-To"] = replyToAddress
-        msg["To"] = toAddress
-        msg["Date"] = rfc822date()
-        msgId = messageid()
-        msg["Message-ID"] = msgId
-
-        msgAlt = MIMEMultipart("alternative")
-        msg.attach(msgAlt)
-
-        # plain version
-        msgPlain = MIMEText(plainText, "plain", "UTF-8")
-        msgAlt.attach(msgPlain)
-
-        # html version
-        msgHtmlRelated = MIMEMultipart("related", type="text/html")
-        msgAlt.attach(msgHtmlRelated)
-
-        msgHtml = MIMEText(htmlText, "html", "UTF-8")
-        msgHtmlRelated.attach(msgHtml)
-
-        # an image for html version
-        if addIcon and iconPath != None and os.path.exists(iconPath):
-
-            with open(iconPath) as iconFile:
-                msgIcon = MIMEImage(iconFile.read(),
-                    _subtype='png;x-apple-mail-type=stationery;name="%s"' %
-                    (iconName,))
-
-            msgIcon.add_header("Content-ID", "<%s>" % (iconName,))
-            msgIcon.add_header("Content-Disposition", "inline;filename=%s" %
-                (iconName,))
-            msgHtmlRelated.attach(msgIcon)
-
-        calendarText = str(calendar)
-        # the icalendar attachment
-        self.log_debug("Mail gateway sending calendar body: %s"
-                       % (calendarText,))
-        msgIcal = MIMEText(calendarText, "calendar", "UTF-8")
-        method = calendar.propertyValue("METHOD").lower()
-        msgIcal.set_param("method", method)
-        msgIcal.add_header("Content-ID", "<invitation.ics>")
-        msgIcal.add_header("Content-Disposition",
-            "inline;filename=invitation.ics")
-        msg.attach(msgIcal)
-
-        return msgId, msg.as_string()
-
-
-    def renderPlainText(self, details, (orgCN, orgEmail), attendees, canceled):
-        """
-        Render text/plain message part based on invitation details and a flag
-        indicating whether the message is a cancellation.
-
-        @return: UTF-8 encoded text.
-
-        @rtype: C{str}
-        """
-        plainAttendeeList = []
-        for cn, mailto in attendees:
-            if cn:
-                plainAttendeeList.append(cn if not mailto else
-                    "%s <%s>" % (cn, mailto))
-            elif mailto:
-                plainAttendeeList.append("<%s>" % (mailto,))
-
-        details['plainAttendees'] = ", ".join(plainAttendeeList)
-
-        details['plainOrganizer'] = (orgCN if not orgEmail else
-            "%s <%s>" % (orgCN, orgEmail))
-
-        # plain text version
-        if canceled:
-            plainTemplate = plainCancelTemplate
-        else:
-            plainTemplate = plainInviteTemplate
-
-        return (plainTemplate % details).encode("UTF-8")
-
-
-    def renderHTML(self, details, organizer, attendees, canceled):
-        """
-        Render HTML message part based on invitation details and a flag
-        indicating whether the message is a cancellation.
-
-        @return: a 2-tuple of (should add icon (C{bool}), html text (C{str},
-            representing utf-8 encoded bytes)).  The first element indicates
-            whether the MIME generator needs to add a C{cid:} icon image part to
-            satisfy the HTML links.
-        """
-        orgCN, orgEmail = organizer
-
-        # TODO: htmlAttendees needs to be a separate element with a separate
-        # template fragment.  Luckily that fragment is the same regardless
-        # of the rest of the template.
-        htmlAttendees = []
-        first = True
-        for cn, mailto in attendees:
-            if not first:
-                htmlAttendees.append(u", ")
-            else:
-                first = False
-
-            if mailto:
-                if not cn:
-                    cn = mailto
-                htmlAttendees.append(
-                    tags.a(href="mailto:%s" % (mailto,))(cn)
-                )
-            else:
-                htmlAttendees.append(cn)
-
-        details['htmlAttendees'] = htmlAttendees
-
-        # TODO: htmlOrganizer is also some HTML that requires additional
-        # template stuff, and once again, it's just a 'mailto:'.
-        # tags.a(href="mailto:"+email)[cn]
-        if orgEmail:
-            details['htmlOrganizer'] = tags.a(href="mailto:%s" % (orgEmail,))(
-                orgCN)
-        else:
-            details['htmlOrganizer'] = orgCN
-
-        templateDir = config.Scheduling.iMIP.MailTemplatesDirectory.rstrip("/")
-        templateName = "cancel.html" if canceled else "invite.html"
-        templatePath = os.path.join(templateDir, templateName)
-
-        if not os.path.exists(templatePath):
-            # Fall back to built-in simple templates:
-            if canceled:
-                htmlTemplate = htmlCancelTemplate
-            else:
-                htmlTemplate = htmlInviteTemplate
-        else: # HTML template file exists
-
-            with open(templatePath) as templateFile:
-                htmlTemplate = templateFile.read()
-
-        class EmailElement(Element):
-            loader = StringFormatTemplateLoader(lambda : StringIO(htmlTemplate),
-                                                "email")
-
-            @renderer
-            def email(self, request, tag):
-                return tag.fillSlots(**details)
-
-        textCollector = []
-        flattenString(None, EmailElement()).addCallback(textCollector.append)
-        htmlText = textCollector[0]
-
-        # If the template refers to an icon in a cid: link, it needs to be added
-        # in the MIME.
-        addIcon = (htmlTemplate.find("cid:%(iconName)s") != -1)
-        return (addIcon, htmlText)
-
-
-    def getEventDetails(self, calendar, language='en'):
-        """
-        Create a dictionary mapping slot names - specifically: summary,
-        description, location, dateInfo, timeInfo, durationInfo, recurrenceInfo,
-        url - with localized string values that should be placed into the HTML
-        and plain-text templates.
-
-        @param calendar: a L{Component} upon which to base the language.
-        @type calendar: L{Component}
-
-        @param language: a 2-letter language code.
-        @type language: C{str}
-
-        @return: a mapping from template slot name to localized text.
-        @rtype: a C{dict} mapping C{bytes} to C{unicode}.
-        """
-
-        # Get the most appropriate component
-        component = calendar.masterComponent()
-        if component is None:
-            component = calendar.mainComponent(True)
-
-        results = {}
-
-        dtStart = component.propertyValue('DTSTART')
-        results['month'] = dtStart.getMonth()
-        results['day'] = dtStart.getDay()
-
-        for propertyToResult in ['summary', 'description', 'location', 'url']:
-            result = component.propertyValue(propertyToResult.upper())
-            if result is None:
-                result = u""
-            else:
-                result = result.decode('utf-8')
-            results[propertyToResult] = result
-
-        with translationTo(language) as trans:
-            results['dateInfo'] = trans.date(component).decode('utf-8')
-            results['timeInfo'], duration = (x.decode('utf-8') for x in trans.time(component))
-            results['durationInfo'] = u"(%s)" % (duration,) if duration else u""
-
-            for propertyName in ('RRULE', 'RDATE', 'EXRULE', 'EXDATE',
-                                 'RECURRENCE-ID'):
-                if component.hasProperty(propertyName):
-                    results['recurrenceInfo'] = _("(Repeating)").decode('utf-8')
-                    break
-            else:
-                results['recurrenceInfo'] = u""
-
-        return results
-
-
-
-#
-# POP3
-#
-
-class POP3Service(service.Service, LoggingMixIn):
-
-    def __init__(self, settings, mailer):
-        if settings["UseSSL"]:
-            self.client = internet.SSLClient(settings["Server"],
-                settings["Port"],
-                POP3DownloadFactory(settings, mailer),
-                ssl.ClientContextFactory())
-        else:
-            self.client = internet.TCPClient(settings["Server"],
-                settings["Port"],
-                POP3DownloadFactory(settings, mailer))
-
-        self.mailer = mailer
-
-
-    def startService(self):
-        self.client.startService()
-
-
-    def stopService(self):
-        self.client.stopService()
-
-
-
-class POP3DownloadProtocol(pop3client.POP3Client, LoggingMixIn):
-    allowInsecureLogin = False
-
-    def serverGreeting(self, greeting):
-        self.log_debug("POP servergreeting")
-        pop3client.POP3Client.serverGreeting(self, greeting)
-        login = self.login(self.factory.settings["Username"],
-            self.factory.settings["Password"])
-        login.addCallback(self.cbLoggedIn)
-        login.addErrback(self.cbLoginFailed)
-
-
-    def cbLoginFailed(self, reason):
-        self.log_error("POP3 login failed for %s" %
-            (self.factory.settings["Username"],))
-        return self.quit()
-
-
-    def cbLoggedIn(self, result):
-        self.log_debug("POP loggedin")
-        return self.listSize().addCallback(self.cbGotMessageSizes)
-
-
-    def cbGotMessageSizes(self, sizes):
-        self.log_debug("POP gotmessagesizes")
-        downloads = []
-        for i in range(len(sizes)):
-            downloads.append(self.retrieve(i).addCallback(self.cbDownloaded, i))
-        return defer.DeferredList(downloads).addCallback(self.cbFinished)
-
-
-    def cbDownloaded(self, lines, id):
-        self.log_debug("POP downloaded message %d" % (id,))
-        self.factory.handleMessage("\r\n".join(lines))
-        self.log_debug("POP deleting message %d" % (id,))
-        self.delete(id)
-
-
-    def cbFinished(self, results):
-        self.log_debug("POP finished")
-        return self.quit()
-
-
-
-class POP3DownloadFactory(protocol.ClientFactory, LoggingMixIn):
-    protocol = POP3DownloadProtocol
-
-    def __init__(self, settings, mailer, reactor=None):
-        self.settings = settings
-        self.mailer = mailer
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-        self.nextPoll = None
-        self.noisy = False
-
-
-    def retry(self, connector=None):
-        # TODO: if connector is None:
-
-        if connector is None:
-            if self.connector is None:
-                self.log_error("No connector to retry")
-                return
-            else:
-                connector = self.connector
-
-        def reconnector():
-            self.nextPoll = None
-            connector.connect()
-
-        self.log_debug("Scheduling next POP3 poll")
-        self.nextPoll = self.reactor.callLater(self.settings["PollingSeconds"],
-            reconnector)
-
-
-    def clientConnectionLost(self, connector, reason):
-        self.connector = connector
-        self.log_debug("POP factory connection lost")
-        self.retry(connector)
-
-
-    def clientConnectionFailed(self, connector, reason):
-        self.connector = connector
-        self.log_info("POP factory connection failed")
-        self.retry(connector)
-
-
-    def handleMessage(self, message):
-        self.log_debug("POP factory handle message")
-        self.log_debug(message)
-
-        return self.mailer.inbound(message)
-
-
-
-#
-# IMAP4
-#
-
-class IMAP4Service(service.Service):
-
-    def __init__(self, settings, mailer):
-
-        if settings["UseSSL"]:
-            self.client = internet.SSLClient(settings["Server"],
-                settings["Port"],
-                IMAP4DownloadFactory(settings, mailer),
-                ssl.ClientContextFactory())
-        else:
-            self.client = internet.TCPClient(settings["Server"],
-                settings["Port"],
-                IMAP4DownloadFactory(settings, mailer))
-
-        self.mailer = mailer
-
-
-    def startService(self):
-        self.client.startService()
-
-
-    def stopService(self):
-        self.client.stopService()
-
-
-
-class IMAP4DownloadProtocol(imap4.IMAP4Client, LoggingMixIn):
-
-    def serverGreeting(self, capabilities):
-        self.log_debug("IMAP servergreeting")
-        return self.authenticate(self.factory.settings["Password"]
-            ).addCallback(self.cbLoggedIn
-            ).addErrback(self.ebAuthenticateFailed)
-
-
-    def ebLogError(self, error):
-        self.log_error("IMAP Error: %s" % (error,))
-
-
-    def ebAuthenticateFailed(self, reason):
-        self.log_debug("IMAP authenticate failed for %s, trying login" %
-            (self.factory.settings["Username"],))
-        return self.login(self.factory.settings["Username"],
-            self.factory.settings["Password"]
-            ).addCallback(self.cbLoggedIn
-            ).addErrback(self.ebLoginFailed)
-
-
-    def ebLoginFailed(self, reason):
-        self.log_error("IMAP login failed for %s" %
-            (self.factory.settings["Username"],))
-        self.transport.loseConnection()
-
-
-    def cbLoggedIn(self, result):
-        self.log_debug("IMAP logged in [%s]" % (self.state,))
-        self.select("Inbox").addCallback(self.cbInboxSelected)
-
-
-    def cbInboxSelected(self, result):
-        self.log_debug("IMAP Inbox selected [%s]" % (self.state,))
-        allMessages = imap4.MessageSet(1, None)
-        self.fetchUID(allMessages, True).addCallback(self.cbGotUIDs)
-
-
-    def cbGotUIDs(self, results):
-        self.log_debug("IMAP got uids [%s]" % (self.state,))
-        self.messageUIDs = [result['UID'] for result in results.values()]
-        self.messageCount = len(self.messageUIDs)
-        self.log_debug("IMAP Inbox has %d messages" % (self.messageCount,))
-        if self.messageCount:
-            self.fetchNextMessage()
-        else:
-            # No messages; close it out
-            self.close().addCallback(self.cbClosed)
-
-
-    def fetchNextMessage(self):
-        self.log_debug("IMAP in fetchnextmessage [%s]" % (self.state,))
-        if self.messageUIDs:
-            nextUID = self.messageUIDs.pop(0)
-            messageListToFetch = imap4.MessageSet(nextUID)
-            self.log_debug("Downloading message %d of %d (%s)" %
-                (self.messageCount - len(self.messageUIDs), self.messageCount,
-                nextUID))
-            self.fetchMessage(messageListToFetch, True).addCallback(
-                self.cbGotMessage, messageListToFetch).addErrback(
-                    self.ebLogError)
-        else:
-            self.log_debug("Seeing if anything new has arrived")
-            # Go back and see if any more messages have come in
-            self.expunge().addCallback(self.cbInboxSelected)
-
-
-    def cbGotMessage(self, results, messageList):
-        self.log_debug("IMAP in cbGotMessage [%s]" % (self.state,))
-        try:
-            messageData = results.values()[0]['RFC822']
-        except IndexError:
-            # results will be empty unless the "twistedmail-imap-flags-anywhere"
-            # patch from http://twistedmatrix.com/trac/ticket/1105 is applied
-            self.log_error("Skipping empty results -- apply twisted patch!")
-            self.fetchNextMessage()
-            return
-
-        d = self.factory.handleMessage(messageData)
-        if isinstance(d, defer.Deferred):
-            d.addCallback(self.cbFlagDeleted, messageList)
-        else:
-            # No deferred returned, so no need for addCallback( )
-            self.cbFlagDeleted(None, messageList)
-
-
-    def cbFlagDeleted(self, results, messageList):
-        self.addFlags(messageList, ("\\Deleted",),
-            uid=True).addCallback(self.cbMessageDeleted, messageList)
-
-
-    def cbMessageDeleted(self, results, messageList):
-        self.log_debug("IMAP in cbMessageDeleted [%s]" % (self.state,))
-        self.log_debug("Deleted message")
-        self.fetchNextMessage()
-
-
-    def cbClosed(self, results):
-        self.log_debug("IMAP in cbClosed [%s]" % (self.state,))
-        self.log_debug("Mailbox closed")
-        self.logout().addCallback(
-            lambda _: self.transport.loseConnection())
-
-
-    def rawDataReceived(self, data):
-        self.log_debug("RAW RECEIVED: %s" % (data,))
-        imap4.IMAP4Client.rawDataReceived(self, data)
-
-
-    def lineReceived(self, line):
-        self.log_debug("RECEIVED: %s" % (line,))
-        imap4.IMAP4Client.lineReceived(self, line)
-
-
-    def sendLine(self, line):
-        self.log_debug("SENDING: %s" % (line,))
-        imap4.IMAP4Client.sendLine(self, line)
-
-
-
-class IMAP4DownloadFactory(protocol.ClientFactory, LoggingMixIn):
-    protocol = IMAP4DownloadProtocol
-
-    def __init__(self, settings, mailer, reactor=None):
-        self.log_debug("Setting up IMAPFactory")
-
-        self.settings = settings
-        self.mailer = mailer
-        if reactor is None:
-            from twisted.internet import reactor
-        self.reactor = reactor
-        self.noisy = False
-
-
-    def buildProtocol(self, addr):
-        p = protocol.ClientFactory.buildProtocol(self, addr)
-        username = self.settings["Username"]
-        p.registerAuthenticator(imap4.CramMD5ClientAuthenticator(username))
-        p.registerAuthenticator(imap4.LOGINAuthenticator(username))
-        p.registerAuthenticator(imap4.PLAINAuthenticator(username))
-        return p
-
-
-    def handleMessage(self, message):
-        self.log_debug("IMAP factory handle message")
-        self.log_debug(message)
-
-        return self.mailer.inbound(message)
-
-
-    def retry(self, connector=None):
-        # TODO: if connector is None:
-
-        if connector is None:
-            if self.connector is None:
-                self.log_error("No connector to retry")
-                return
-            else:
-                connector = self.connector
-
-        def reconnector():
-            self.nextPoll = None
-            connector.connect()
-
-        self.log_debug("Scheduling next IMAP4 poll")
-        self.nextPoll = self.reactor.callLater(self.settings["PollingSeconds"],
-            reconnector)
-
-
-    def clientConnectionLost(self, connector, reason):
-        self.connector = connector
-        self.log_debug("IMAP factory connection lost")
-        self.retry(connector)
-
-
-    def clientConnectionFailed(self, connector, reason):
-        self.connector = connector
-        self.log_warn("IMAP factory connection failed")
-        self.retry(connector)
+    @param store: The store to copy tokens into
+    @type store: L{CommonDataStore}
+    """
+    oldDB = MailGatewayTokensDatabase(path)
+    txn = store.newTransaction()
+    for token, organizer, attendee, icaluid in oldDB.getAllTokens():
+        yield txn.imipCreateToken(organizer, attendee, icaluid, token=token)
+    yield txn.commit()
+    os.remove(oldDB.dbpath)

Added: CalendarServer/trunk/twistedcaldav/scheduling/imip/outbound.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/outbound.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/outbound.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,789 @@
+##
+# Copyright (c) 2005-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Outbound IMIP mail handling for Calendar Server
+"""
+
+from __future__ import with_statement
+
+from cStringIO import StringIO
+import os
+
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import email.utils
+from pycalendar.datetime import PyCalendarDateTime
+from pycalendar.duration import PyCalendarDuration
+from twext.enterprise.dal.record import fromTable
+from twext.enterprise.queue import WorkItem
+from twext.python.log import Logger, LoggingMixIn
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.mail.smtp import messageid, rfc822date
+from twisted.web.microdom import Text as DOMText, Element as DOMElement
+from twisted.web.microdom import parseString
+from twisted.web.template import XMLString, TEMPLATE_NAMESPACE, Element, renderer, flattenString, tags
+from twistedcaldav.config import config
+from twistedcaldav.ical import Component
+from twistedcaldav.localization import translationTo, _
+from twistedcaldav.scheduling.cuaddress import normalizeCUAddr
+from twistedcaldav.scheduling.imip.smtpsender import SMTPSender
+from txdav.common.datastore.sql_tables import schema
+
+
+
+log = Logger()
+
+
+""" SCHEMA:
+create sequence WORKITEM_SEQ;
+
+create table IMIP_INVITATION_WORK (
+  WORK_ID         integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE      timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  FROM_ADDR       varchar(255) not null,
+  TO_ADDR         varchar(255) not null,
+  ICALENDAR_TEXT  text         not null
+);
+"""
+
+
+class IMIPInvitationWork(WorkItem, fromTable(schema.IMIP_INVITATION_WORK)):
+    """
+    Sends outbound IMIP messages
+    """
+
+    mailSender = None
+
+    @classmethod
+    def getMailSender(cls):
+        """
+        Instantiate and return a singleton MailSender object
+        @return: a MailSender
+        """
+        if cls.mailSender is None:
+            if config.Scheduling.iMIP.Enabled:
+                settings = config.Scheduling.iMIP.Sending
+                smtpSender = SMTPSender(settings.Username, settings.Password,
+                    settings.UseSSL, settings.Server, settings.Port)
+                cls.mailSender = MailSender(settings.Address,
+                    settings.SuppressionDays, smtpSender=smtpSender)
+        return cls.mailSender
+
+    @inlineCallbacks
+    def doWork(self):
+        """
+        Send an outbound IMIP message
+        """
+        mailSender = self.getMailSender()
+        if mailSender is not None:
+            calendar = Component.fromString(self.icalendarText)
+            yield mailSender.outbound(self.transaction,
+                self.fromAddr, self.toAddr, calendar)
+
+#
+# Templates
+#
+
+plainCancelTemplate = u"""%(subject)s
+
+%(orgLabel)s: %(plainOrganizer)s
+%(dateLabel)s: %(dateInfo)s %(recurrenceInfo)s
+%(timeLabel)s: %(timeInfo)s %(durationInfo)s
+"""
+
+plainInviteTemplate = u"""%(subject)s
+
+%(orgLabel)s: %(plainOrganizer)s
+%(locLabel)s: %(location)s
+%(dateLabel)s: %(dateInfo)s %(recurrenceInfo)s
+%(timeLabel)s: %(timeInfo)s %(durationInfo)s
+%(descLabel)s: %(description)s
+%(urlLabel)s: %(url)s
+%(attLabel)s: %(plainAttendees)s
+"""
+
+
+htmlCancelTemplate = u"""<html>
+    <body><div>
+
+    <h1>%(subject)s</h1>
+    <p>
+    <h3>%(orgLabel)s:</h3> %(htmlOrganizer)s
+    </p>
+    <p>
+    <h3>%(dateLabel)s:</h3> %(dateInfo)s %(recurrenceInfo)s
+    </p>
+    <p>
+    <h3>%(timeLabel)s:</h3> %(timeInfo)s %(durationInfo)s
+    </p>
+    """.encode("utf-8")
+
+
+htmlInviteTemplate = u"""<html>
+    <body><div>
+    <p>%(inviteLabel)s</p>
+
+    <h1>%(summary)s</h1>
+    <p>
+    <h3>%(orgLabel)s:</h3> %(htmlOrganizer)s
+    </p>
+    <p>
+    <h3>%(locLabel)s:</h3> %(location)s
+    </p>
+    <p>
+    <h3>%(dateLabel)s:</h3> %(dateInfo)s %(recurrenceInfo)s
+    </p>
+    <p>
+    <h3>%(timeLabel)s:</h3> %(timeInfo)s %(durationInfo)s
+    </p>
+    <p>
+    <h3>%(descLabel)s:</h3> %(description)s
+    </p>
+    <p>
+    <h3>%(urlLabel)s:</h3> <a href="%(url)s">%(url)s</a>
+    </p>
+    <p>
+    <h3>%(attLabel)s:</h3> %(htmlAttendees)s
+    </p>
+    """.encode("utf-8")
+
+def _visit(document, node):
+    if isinstance(node, DOMText):
+        idx = node.parentNode.childNodes.index(node)
+        splitted = node.data.split("%(")
+        firstTextNode = document.createTextNode(splitted[0])
+        firstTextNode.parentNode = node.parentNode
+        replacements = [firstTextNode]
+        for moreText in splitted[1:]:
+            slotName, extra = moreText.split(')', 1)
+            extra = extra[1:]
+            slotElement = document.createElement('t:slot')
+            slotElement.setAttribute("name", slotName)
+            slotElement.parentNode = node.parentNode
+            textNode = document.createTextNode(extra)
+            textNode.parentNode = node.parentNode
+            replacements.append(slotElement)
+            replacements.append(textNode)
+        node.parentNode.childNodes[idx:idx + 1] = replacements
+
+    elif isinstance(node, DOMElement):
+        for attrName, attrVal in node.attributes.items():
+            if '%(' in attrVal:
+                del node.attributes[attrName]
+                elem = document.createElement('t:attr')
+                elem.setAttribute('name', attrName)
+                textNode = document.createTextNode(attrVal)
+                elem.appendChild(textNode)
+                node.appendChild(elem)
+
+
+
+def _walk(document, n):
+    _visit(document, n)
+    for subn in n.childNodes:
+        _walk(document, subn)
+
+
+
+def _fixup(data, rendererName):
+    document = parseString(data, beExtremelyLenient=True)
+    document.documentElement.setAttribute(
+        "xmlns:t", TEMPLATE_NAMESPACE
+    )
+    document.doctype = (
+        'html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" '
+        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
+    )
+    document.documentElement.setAttribute(
+        "t:render", rendererName
+    )
+    _walk(document, document)
+    result = document.toxml()
+    return result
+
+
+
+class StringFormatTemplateLoader(object):
+    """
+    Loader for twisted.web.template that converts a template with %()s slots.
+    """
+    def __init__(self, fileFactory, rendererName):
+        """
+        @param fileFactory: a 1-argument callable which returns a file-like
+            object that contains the %()s-format template.
+
+        @param rendererName: the name of the renderer.
+
+        @type rendererName: C{str}
+        """
+        self.fileFactory = fileFactory
+        self.rendererName = rendererName
+
+
+    def load(self):
+        html = _fixup(self.fileFactory().read(), self.rendererName)
+        return XMLString(html).load()
+
+
+
+def localizedLabels(language, canceled, inviteState):
+    """
+    Generate localized labels for an email in the given language.
+
+    @param language: a 2-letter language code
+
+    @type language: C{str}
+
+    @return: a 2-tuple of (subjectFormatString, labelDict), where the first is a
+        format string for use in the subject, and the latter is a dictionary
+        with labels suitable for filling out HTML and plain-text templates.  All
+        values are C{str}s.
+    """
+    with translationTo(language):
+        if canceled:
+            subjectFormatString = _("Event canceled: %(summary)s")
+        elif inviteState == "new":
+            subjectFormatString = _("Event invitation: %(summary)s")
+        elif inviteState == "update":
+            subjectFormatString = _("Event update: %(summary)s")
+        else:
+            subjectFormatString = _("Event reply: %(summary)s")
+
+        if canceled:
+            inviteLabel = _("Event Canceled")
+        else:
+            if inviteState == "new":
+                inviteLabel = _("Event Invitation")
+            elif inviteState == "update":
+                inviteLabel = _("Event Update")
+            else:
+                inviteLabel = _("Event Reply")
+
+        labels = dict(
+            dateLabel=_("Date"),
+            timeLabel=_("Time"),
+            durationLabel=_("Duration"),
+            recurrenceLabel=_("Occurs"),
+            descLabel=_("Description"),
+            urlLabel=_("URL"),
+            orgLabel=_("Organizer"),
+            attLabel=_("Attendees"),
+            locLabel=_("Location"),
+            inviteLabel=inviteLabel,
+        )
+
+        # The translations we get back from gettext are utf-8 encoded
+        # strings, so convert to unicode
+        for key in labels.keys():
+            if isinstance(labels[key], str):
+                labels[key] = labels[key].decode("utf-8")
+
+    return subjectFormatString.decode("utf-8"), labels
+
+
+
+class MailSender(LoggingMixIn):
+    """
+    Generates outbound IMIP messages and sends them.
+    """
+
+    def __init__(self, address, suppressionDays, smtpSender):
+        self.address = address
+        self.suppressionDays = suppressionDays
+        self.smtpSender = smtpSender
+
+
+    @inlineCallbacks
+    def outbound(self, txn, originator, recipient, calendar, language='en',
+        onlyAfter=None):
+        """
+        Generates and sends an outbound IMIP message.
+
+        @param txn: the transaction to use for looking up/creating tokens
+        @type txn: L{CommonStoreTransaction}
+        """
+
+        if onlyAfter is None:
+            duration = PyCalendarDuration(days=self.suppressionDays)
+            onlyAfter = PyCalendarDateTime.getNowUTC() - duration
+
+        component = calendar.masterComponent()
+        if component is None:
+            component = calendar.mainComponent(True)
+        icaluid = component.propertyValue("UID")
+        method = calendar.propertyValue("METHOD")
+
+        # Clean up the attendee list which is purely used within the human
+        # readable email message (not modifying the calendar body)
+        attendees = []
+        for attendeeProp in calendar.getAllAttendeeProperties():
+            cutype = attendeeProp.parameterValue("CUTYPE", "INDIVIDUAL")
+            if cutype == "INDIVIDUAL":
+                cn = attendeeProp.parameterValue("CN", None)
+                if cn is not None:
+                    cn = cn.decode("utf-8")
+                cuaddr = normalizeCUAddr(attendeeProp.value())
+                if cuaddr.startswith("mailto:"):
+                    mailto = cuaddr[7:]
+                    if not cn:
+                        cn = mailto
+                else:
+                    emailAddress = attendeeProp.parameterValue("EMAIL", None)
+                    if emailAddress:
+                        mailto = emailAddress
+                    else:
+                        mailto = None
+
+                if cn or mailto:
+                    attendees.append((cn, mailto))
+
+        toAddr = recipient
+        if not recipient.lower().startswith("mailto:"):
+            raise ValueError("ATTENDEE address '%s' must be mailto: for iMIP "
+                             "operation." % (recipient,))
+        recipient = recipient[7:]
+
+        if method != "REPLY":
+            # Invites and cancellations:
+
+            # Reuse or generate a token based on originator, toAddr, and
+            # event uid
+            token = (yield txn.imipGetToken(originator, toAddr.lower(), icaluid))
+            if token is None:
+
+                # Because in the past the originator was sometimes in mailto:
+                # form, lookup an existing token by mailto: as well
+                organizerProperty = calendar.getOrganizerProperty()
+                organizerEmailAddress = organizerProperty.parameterValue("EMAIL", None)
+                if organizerEmailAddress is not None:
+                    token = (yield txn.imipGetToken("mailto:%s" % (organizerEmailAddress.lower(),), toAddr.lower(), icaluid))
+
+            if token is None:
+                token = (yield txn.imipCreateToken(originator, toAddr.lower(), icaluid))
+                self.log_debug("Mail gateway created token %s for %s "
+                               "(originator), %s (recipient) and %s (icaluid)"
+                               % (token, originator, toAddr, icaluid))
+                inviteState = "new"
+
+            else:
+                self.log_debug("Mail gateway reusing token %s for %s "
+                               "(originator), %s (recipient) and %s (icaluid)"
+                               % (token, originator, toAddr, icaluid))
+                inviteState = "update"
+
+            fullServerAddress = self.address
+            _ignore_name, serverAddress = email.utils.parseaddr(fullServerAddress)
+            pre, post = serverAddress.split('@')
+            addressWithToken = "%s+%s@%s" % (pre, token, post)
+
+            organizerProperty = calendar.getOrganizerProperty()
+            organizerEmailAddress = organizerProperty.parameterValue("EMAIL",
+                                                                     None)
+            organizerValue = organizerProperty.value()
+            organizerProperty.setValue("mailto:%s" % (addressWithToken,))
+
+            # If the organizer is also an attendee, update that attendee value
+            # to match
+            organizerAttendeeProperty = calendar.getAttendeeProperty(
+                [organizerValue])
+            if organizerAttendeeProperty is not None:
+                organizerAttendeeProperty.setValue("mailto:%s" %
+                                                   (addressWithToken,))
+
+            # The email's From will include the originator's real name email
+            # address if available.  Otherwise it will be the server's email
+            # address (without # + addressing)
+            if organizerEmailAddress:
+                orgEmail = fromAddr = organizerEmailAddress
+            else:
+                fromAddr = serverAddress
+                orgEmail = None
+            cn = calendar.getOrganizerProperty().parameterValue('CN', None)
+            if cn is None:
+                cn = u'Calendar Server'
+                orgCN = orgEmail
+            else:
+                orgCN = cn = cn.decode("utf-8")
+
+            # a unicode cn (rather than an encode string value) means the
+            # from address will get properly encoded per rfc2047 within the
+            # MIMEMultipart in generateEmail
+            formattedFrom = "%s <%s>" % (cn, fromAddr)
+
+            # Reply-to address will be the server+token address
+
+        else: # REPLY
+            inviteState = "reply"
+
+            # Look up the attendee property corresponding to the originator
+            # of this reply
+            originatorAttendeeProperty = calendar.getAttendeeProperty(
+                [originator])
+            formattedFrom = fromAddr = originator = ""
+            if originatorAttendeeProperty:
+                originatorAttendeeEmailAddress = (
+                    originatorAttendeeProperty.parameterValue("EMAIL", None)
+                )
+                if originatorAttendeeEmailAddress:
+                    formattedFrom = fromAddr = originator = (
+                        originatorAttendeeEmailAddress
+                    )
+
+            organizerMailto = str(calendar.getOrganizer())
+            if not organizerMailto.lower().startswith("mailto:"):
+                raise ValueError("ORGANIZER address '%s' must be mailto: "
+                                 "for REPLY." % (organizerMailto,))
+            orgEmail = organizerMailto[7:]
+
+            orgCN = calendar.getOrganizerProperty().parameterValue('CN', None)
+            addressWithToken = formattedFrom
+
+        # At the point we've created the token in the db, which we always
+        # want to do, but if this message is for an event completely in
+        # the past we don't want to actually send an email.
+        if not calendar.hasInstancesAfter(onlyAfter):
+            self.log_debug("Skipping IMIP message for old event")
+            returnValue(True)
+
+        # Now prevent any "internal" CUAs from being exposed by converting
+        # to mailto: if we have one
+        for attendeeProp in calendar.getAllAttendeeProperties():
+            cutype = attendeeProp.parameterValue('CUTYPE', None)
+            if cutype == "INDIVIDUAL":
+                cuaddr = normalizeCUAddr(attendeeProp.value())
+                if not cuaddr.startswith("mailto:"):
+                    emailAddress = attendeeProp.parameterValue("EMAIL", None)
+                    if emailAddress:
+                        attendeeProp.setValue("mailto:%s" % (emailAddress,))
+
+        msgId, message = self.generateEmail(inviteState, calendar, orgEmail,
+            orgCN, attendees, formattedFrom, addressWithToken, recipient,
+            language=language)
+
+        try:
+            success = (yield self.smtpSender.sendMessage(fromAddr, toAddr,
+                msgId, message))
+            returnValue(success)
+        except Exception, e:
+            self.log_error("Failed to send IMIP message (%s)" % (str(e),))
+            returnValue(False)
+
+
+    def getIconPath(self, details, canceled, language='en'):
+        iconDir = config.Scheduling.iMIP.MailIconsDirectory.rstrip("/")
+
+        if canceled:
+            iconName = "canceled.png"
+            iconPath = os.path.join(iconDir, iconName)
+            if os.path.exists(iconPath):
+                return iconPath
+            else:
+                return None
+
+        else:
+            month = int(details['month'])
+            day = int(details['day'])
+            with translationTo(language) as trans:
+                monthName = trans.monthAbbreviation(month)
+            iconName = "%02d.png" % (day,)
+            iconPath = os.path.join(iconDir, monthName.encode("utf-8"), iconName)
+            if not os.path.exists(iconPath):
+                # Try the generic (numeric) version
+                iconPath = os.path.join(iconDir, "%02d" % (month,), iconName)
+                if not os.path.exists(iconPath):
+                    return None
+            return iconPath
+
+
+    def generateEmail(self, inviteState, calendar, orgEmail, orgCN,
+                      attendees, fromAddress, replyToAddress, toAddress,
+                      language='en'):
+        """
+        Generate MIME text containing an iMIP invitation, cancellation, update
+        or reply.
+
+        @param inviteState: 'new', 'update', or 'reply'.
+
+        @type inviteState: C{str}
+
+        @param calendar: the iCalendar component to attach to the email.
+
+        @type calendar: L{twistedcaldav.ical.Component}
+
+        @param orgEmail: The email for the organizer, in C{localhost at domain}
+            format, or C{None} if the organizer has no email address.
+
+        @type orgEmail: C{str} or C{NoneType}
+
+        @param orgCN: Common name / display name for the organizer.
+
+        @type orgCN: C{unicode}
+
+        @param attendees: A C{list} of 2-C{tuple}s of (common name, email
+            address) similar to (orgEmail, orgCN).
+
+        @param fromAddress: the address to use in the C{From:} header of the
+            email.
+
+        @type fromAddress: C{str}
+
+        @param replyToAddress: the address to use in the C{Reply-To} header.
+
+        @type replyToAddress: C{str}
+
+        @param toAddress: the address to use in the C{To} header.
+
+        @type toAddress: C{str}
+
+        @param language: a 2-letter language code describing the target
+            language that the email should be generated in.
+
+        @type language: C{str}
+
+        @return: a 2-tuple of C{str}s: (message ID, message text).  The message
+            ID is the value of the C{Message-ID} header, and the message text is
+            the full MIME message, ready for transport over SMTP.
+        """
+
+        details = self.getEventDetails(calendar, language=language)
+        canceled = (calendar.propertyValue("METHOD") == "CANCEL")
+        iconPath = self.getIconPath(details, canceled, language=language)
+
+        subjectFormat, labels = localizedLabels(language, canceled, inviteState)
+        details.update(labels)
+
+        details['subject'] = subjectFormat % {'summary' : details['summary']}
+        details['iconName'] = iconName = "calicon.png"
+
+        plainText = self.renderPlainText(details, (orgCN, orgEmail),
+                                         attendees, canceled)
+
+        [addIcon, htmlText] = self.renderHTML(details, (orgCN, orgEmail),
+                                              attendees, canceled)
+
+        msg = MIMEMultipart()
+        msg["From"] = fromAddress
+        msg["Subject"] = details['subject']
+        msg["Reply-To"] = replyToAddress
+        msg["To"] = toAddress
+        msg["Date"] = rfc822date()
+        msgId = messageid()
+        msg["Message-ID"] = msgId
+
+        msgAlt = MIMEMultipart("alternative")
+        msg.attach(msgAlt)
+
+        # plain version
+        msgPlain = MIMEText(plainText, "plain", "UTF-8")
+        msgAlt.attach(msgPlain)
+
+        # html version
+        msgHtmlRelated = MIMEMultipart("related", type="text/html")
+        msgAlt.attach(msgHtmlRelated)
+
+        msgHtml = MIMEText(htmlText, "html", "UTF-8")
+        msgHtmlRelated.attach(msgHtml)
+
+        # an image for html version
+        if addIcon and iconPath != None and os.path.exists(iconPath):
+
+            with open(iconPath) as iconFile:
+                msgIcon = MIMEImage(iconFile.read(),
+                    _subtype='png;x-apple-mail-type=stationery;name="%s"' %
+                    (iconName,))
+
+            msgIcon.add_header("Content-ID", "<%s>" % (iconName,))
+            msgIcon.add_header("Content-Disposition", "inline;filename=%s" %
+                (iconName,))
+            msgHtmlRelated.attach(msgIcon)
+
+        calendarText = str(calendar)
+        # the icalendar attachment
+        self.log_debug("Mail gateway sending calendar body: %s"
+                       % (calendarText,))
+        msgIcal = MIMEText(calendarText, "calendar", "UTF-8")
+        method = calendar.propertyValue("METHOD").lower()
+        msgIcal.set_param("method", method)
+        msgIcal.add_header("Content-ID", "<invitation.ics>")
+        msgIcal.add_header("Content-Disposition",
+            "inline;filename=invitation.ics")
+        msg.attach(msgIcal)
+
+        return msgId, msg.as_string()
+
+
+    def renderPlainText(self, details, (orgCN, orgEmail), attendees, canceled):
+        """
+        Render text/plain message part based on invitation details and a flag
+        indicating whether the message is a cancellation.
+
+        @return: UTF-8 encoded text.
+
+        @rtype: C{str}
+        """
+        plainAttendeeList = []
+        for cn, mailto in attendees:
+            if cn:
+                plainAttendeeList.append(cn if not mailto else
+                    "%s <%s>" % (cn, mailto))
+            elif mailto:
+                plainAttendeeList.append("<%s>" % (mailto,))
+
+        details['plainAttendees'] = ", ".join(plainAttendeeList)
+
+        details['plainOrganizer'] = (orgCN if not orgEmail else
+            "%s <%s>" % (orgCN, orgEmail))
+
+        # plain text version
+        if canceled:
+            plainTemplate = plainCancelTemplate
+        else:
+            plainTemplate = plainInviteTemplate
+
+        return (plainTemplate % details).encode("UTF-8")
+
+
+    def renderHTML(self, details, organizer, attendees, canceled):
+        """
+        Render HTML message part based on invitation details and a flag
+        indicating whether the message is a cancellation.
+
+        @return: a 2-tuple of (should add icon (C{bool}), html text (C{str},
+            representing utf-8 encoded bytes)).  The first element indicates
+            whether the MIME generator needs to add a C{cid:} icon image part to
+            satisfy the HTML links.
+        """
+        orgCN, orgEmail = organizer
+
+        # TODO: htmlAttendees needs to be a separate element with a separate
+        # template fragment.  Luckily that fragment is the same regardless
+        # of the rest of the template.
+        htmlAttendees = []
+        first = True
+        for cn, mailto in attendees:
+            if not first:
+                htmlAttendees.append(u", ")
+            else:
+                first = False
+
+            if mailto:
+                if not cn:
+                    cn = mailto
+                htmlAttendees.append(
+                    tags.a(href="mailto:%s" % (mailto,))(cn)
+                )
+            else:
+                htmlAttendees.append(cn)
+
+        details['htmlAttendees'] = htmlAttendees
+
+        # TODO: htmlOrganizer is also some HTML that requires additional
+        # template stuff, and once again, it's just a 'mailto:'.
+        # tags.a(href="mailto:"+email)[cn]
+        if orgEmail:
+            details['htmlOrganizer'] = tags.a(href="mailto:%s" % (orgEmail,))(
+                orgCN)
+        else:
+            details['htmlOrganizer'] = orgCN
+
+        templateDir = config.Scheduling.iMIP.MailTemplatesDirectory.rstrip("/")
+        templateName = "cancel.html" if canceled else "invite.html"
+        templatePath = os.path.join(templateDir, templateName)
+
+        if not os.path.exists(templatePath):
+            # Fall back to built-in simple templates:
+            if canceled:
+                htmlTemplate = htmlCancelTemplate
+            else:
+                htmlTemplate = htmlInviteTemplate
+        else: # HTML template file exists
+
+            with open(templatePath) as templateFile:
+                htmlTemplate = templateFile.read()
+
+        class EmailElement(Element):
+            loader = StringFormatTemplateLoader(lambda : StringIO(htmlTemplate),
+                                                "email")
+
+            @renderer
+            def email(self, request, tag):
+                return tag.fillSlots(**details)
+
+        textCollector = []
+        flattenString(None, EmailElement()).addCallback(textCollector.append)
+        htmlText = textCollector[0]
+
+        # If the template refers to an icon in a cid: link, it needs to be added
+        # in the MIME.
+        addIcon = (htmlTemplate.find("cid:%(iconName)s") != -1)
+        return (addIcon, htmlText)
+
+
+    def getEventDetails(self, calendar, language='en'):
+        """
+        Create a dictionary mapping slot names - specifically: summary,
+        description, location, dateInfo, timeInfo, durationInfo, recurrenceInfo,
+        url - with localized string values that should be placed into the HTML
+        and plain-text templates.
+
+        @param calendar: a L{Component} upon which to base the language.
+        @type calendar: L{Component}
+
+        @param language: a 2-letter language code.
+        @type language: C{str}
+
+        @return: a mapping from template slot name to localized text.
+        @rtype: a C{dict} mapping C{bytes} to C{unicode}.
+        """
+
+        # Get the most appropriate component
+        component = calendar.masterComponent()
+        if component is None:
+            component = calendar.mainComponent(True)
+
+        results = {}
+
+        dtStart = component.propertyValue('DTSTART')
+        results['month'] = dtStart.getMonth()
+        results['day'] = dtStart.getDay()
+
+        for propertyToResult in ['summary', 'description', 'location', 'url']:
+            result = component.propertyValue(propertyToResult.upper())
+            if result is None:
+                result = u""
+            else:
+                result = result.decode('utf-8')
+            results[propertyToResult] = result
+
+        with translationTo(language) as trans:
+            results['dateInfo'] = trans.date(component).decode('utf-8')
+            results['timeInfo'], duration = (x.decode('utf-8') for x in trans.time(component))
+            results['durationInfo'] = u"(%s)" % (duration,) if duration else u""
+
+            for propertyName in ('RRULE', 'RDATE', 'EXRULE', 'EXDATE',
+                                 'RECURRENCE-ID'):
+                if component.hasProperty(propertyName):
+                    results['recurrenceInfo'] = _("(Repeating)").decode('utf-8')
+                    break
+            else:
+                results['recurrenceInfo'] = u""
+
+        return results
+
+
+

Modified: CalendarServer/trunk/twistedcaldav/scheduling/imip/scheduler.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/scheduler.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/scheduler.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -15,22 +15,17 @@
 ##
 
 from twext.python.log import Logger
-from twext.web2 import responsecode
 from twext.web2.dav.http import ErrorResponse
-from twext.web2.http import HTTPError
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import succeed
 from twistedcaldav.caldavxml import caldav_namespace
-from twistedcaldav.config import config
-from twistedcaldav.scheduling import addressmapping
 from twistedcaldav.scheduling.cuaddress import RemoteCalendarUser
-from twistedcaldav.scheduling.scheduler import RemoteScheduler, \
-    ScheduleResponseQueue
-import itertools
-import socket
+from twistedcaldav.scheduling.scheduler import RemoteScheduler
+from twistedcaldav.scheduling.scheduler import ScheduleResponseQueue
 
 
 """
-L{IMIPScheduler} - handles deliveries for scheduling messages being POSTed to the iMIP inbox.
+L{IMIPScheduler} - handles deliveries for scheduling messages retrieved via
+mail
 """
 
 __all__ = [
@@ -63,26 +58,14 @@
         pass
 
 
-    @inlineCallbacks
     def checkOriginator(self):
         """
-        Check the validity of the Originator header.
+        The originator always comes out of the tokens db
         """
+        self.originator = RemoteCalendarUser(self.originator)
+        return succeed(None)
 
-        # For remote requests we do not allow the originator to be a local user or one within our domain.
-        originatorPrincipal = self.resource.principalForCalendarUserAddress(self.originator)
-        localUser = (yield addressmapping.mapper.isCalendarUserInMyDomain(self.originator))
-        if originatorPrincipal or localUser:
-            log.err("Cannot use originator that is on this server: %s" % (self.originator,))
-            raise HTTPError(self.errorResponse(
-                responsecode.FORBIDDEN,
-                self.errorElements["originator-denied"],
-                "Originator cannot be local to server",
-            ))
-        else:
-            self.originator = RemoteCalendarUser(self.originator)
 
-
     def checkOrganizerAsOriginator(self):
         pass
 
@@ -92,21 +75,4 @@
 
 
     def securityChecks(self):
-        """
-        Check that the connection is from the mail gateway
-        """
-        allowed = config.Scheduling['iMIP']['MailGatewayServer']
-        # Get the request IP and map to hostname.
-        clientip = self.request.remoteAddr.host
-        host, aliases, _ignore_ips = socket.gethostbyaddr(clientip)
-        for host in itertools.chain((host, clientip), aliases):
-            if host == allowed:
-                break
-        else:
-            log.err("Only %s is allowed to submit internal scheduling requests, not %s" % (allowed, host))
-            # TODO: verify this is the right response:
-            raise HTTPError(self.errorResponse(
-                responsecode.FORBIDDEN,
-                self.errorElements["originator-denied"],
-                "Originator server not allowed to send to this server",
-            ))
+        pass

Added: CalendarServer/trunk/twistedcaldav/scheduling/imip/smtpsender.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/smtpsender.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/smtpsender.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,75 @@
+##
+# Copyright (c) 2005-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+SMTP sending utility
+"""
+
+from cStringIO import StringIO
+
+from twext.internet.adaptendpoint import connect
+from twext.internet.gaiendpoint import GAIEndpoint
+from twext.python.log import Logger
+from twisted.internet import defer, ssl, reactor as _reactor
+from twisted.mail.smtp import ESMTPSenderFactory
+
+log = Logger()
+
+class SMTPSender(object):
+
+    def __init__(self, username, password, useSSL, server, port):
+        self.username = username
+        self.password = password
+        self.useSSL = useSSL
+        self.server = server
+        self.port = port
+
+    def sendMessage(self, fromAddr, toAddr, msgId, message):
+
+        log.debug("Sending: %s" % (message,))
+        def _success(result, msgId, fromAddr, toAddr):
+            log.info("Sent IMIP message %s from %s to %s" %
+                (msgId, fromAddr, toAddr))
+            return True
+
+        def _failure(failure, msgId, fromAddr, toAddr):
+            log.error("Failed to send IMIP message %s from %s "
+                           "to %s (Reason: %s)" %
+                           (msgId, fromAddr, toAddr,
+                            failure.getErrorMessage()))
+            return False
+
+        deferred = defer.Deferred()
+
+        if self.useSSL:
+            contextFactory = ssl.ClientContextFactory()
+        else:
+            contextFactory = None
+
+        factory = ESMTPSenderFactory(
+            self.username, self.password,
+            fromAddr, toAddr,
+            # per http://trac.calendarserver.org/ticket/416 ...
+            StringIO(message.replace("\r\n", "\n")), deferred,
+            contextFactory=contextFactory,
+            requireAuthentication=False,
+            requireTransportSecurity=self.useSSL)
+
+        connect(GAIEndpoint(_reactor, self.server, self.port),
+                factory)
+        deferred.addCallback(_success, msgId, fromAddr, toAddr)
+        deferred.addErrback(_failure, msgId, fromAddr, toAddr)
+        return deferred

Added: CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_inbound.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_inbound.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_inbound.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,362 @@
+##
+# Copyright (c) 2008-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+import email
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.modules import getModule
+from twistedcaldav.ical import Component
+from twistedcaldav.scheduling.imip.inbound import MailReceiver
+from twistedcaldav.scheduling.imip.inbound import MailRetriever
+from twistedcaldav.scheduling.imip.inbound import injectMessage
+from twistedcaldav.scheduling.imip.inbound import IMIPReplyWork
+from twistedcaldav.scheduling.itip import iTIPRequestStatus
+from twistedcaldav.test.util import TestCase
+from twistedcaldav.test.util import xmlFile
+from txdav.common.datastore.test.util import buildStore
+from calendarserver.tap.util import getRootResource
+from twistedcaldav.config import config, ConfigDict
+
+
+class InboundTests(TestCase):
+
+    @inlineCallbacks
+    def setUp(self):
+        super(InboundTests, self).setUp()
+
+        self.store = yield buildStore(self, None)
+        self.patch(config.DirectoryService.params, "xmlFile", xmlFile)
+        self.root = getRootResource(config, self.store)
+        self.directory = self.root.getDirectory()
+        self.receiver = MailReceiver(self.store, self.directory)
+        self.retriever = MailRetriever(self.store, self.directory,
+            ConfigDict({
+                "Type" : "pop",
+                "UseSSL" : False,
+                "Server" : "example.com",
+                "Port" : 123,
+            })
+        )
+
+        def decorateTransaction(txn):
+            txn._rootResource = self.root
+            txn._mailRetriever = self.retriever
+
+        self.store.callWithNewTransactions(decorateTransaction)
+        module = getModule(__name__)
+        self.dataPath = module.filePath.sibling("data")
+
+    def dataFile(self, name):
+        """
+        Get the contents of a given data file from the 'data/mail' test
+        fixtures directory.
+        """
+        return self.dataPath.child(name).getContent()
+
+
+    def test_checkDSNFailure(self):
+
+        data = {
+            'good_reply' : (False, None, None),
+            'dsn_failure_no_original' : (True, 'failed', None),
+            'dsn_failure_no_ics' : (True, 'failed', None),
+            'dsn_failure_with_ics' : (True, 'failed', '''BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+METHOD:REQUEST
+PRODID:-//example Inc.//iCal 3.0//EN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+DTSTART:20071104T020000
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+TZNAME:PST
+TZOFFSETFROM:-0700
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+DTSTART:20070311T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+TZNAME:PDT
+TZOFFSETFROM:-0800
+TZOFFSETTO:-0700
+END:DAYLIGHT
+END:VTIMEZONE
+BEGIN:VEVENT
+UID:1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C
+DTSTART;TZID=US/Pacific:20080812T094500
+DTEND;TZID=US/Pacific:20080812T104500
+ATTENDEE;CUTYPE=INDIVIDUAL;CN=User 01;PARTSTAT=ACCEPTED:mailto:user01 at exam
+ ple.com
+ATTENDEE;CUTYPE=INDIVIDUAL;RSVP=TRUE;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-A
+ CTION;CN=nonexistant at example.com:mailto:nonexistant at example.com
+CREATED:20080812T191857Z
+DTSTAMP:20080812T191932Z
+ORGANIZER;CN=User 01:mailto:xyzzy+8e16b897-d544-4217-88e9-a363d08
+ 46f6c at example.com
+SEQUENCE:2
+SUMMARY:New Event
+TRANSP:OPAQUE
+END:VEVENT
+END:VCALENDAR
+'''),
+        }
+
+        for filename, expected in data.iteritems():
+            msg = email.message_from_string(self.dataFile(filename))
+            self.assertEquals(self.receiver.checkDSN(msg), expected)
+
+
+    @inlineCallbacks
+    def test_processDSN(self):
+
+        template = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+METHOD:REQUEST
+PRODID:-//example Inc.//iCal 3.0//EN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:DAYLIGHT
+DTSTART:20070311T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+TZNAME:PDT
+TZOFFSETFROM:-0800
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20071104T020000
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+TZNAME:PST
+TZOFFSETFROM:-0700
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+UID:1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C
+DTSTART;TZID=US/Pacific:20080812T094500
+DTEND;TZID=US/Pacific:20080812T104500
+ATTENDEE;CUTYPE=INDIVIDUAL;CN=User 01;PARTSTAT=ACCEPTED:mailto:user01 at exam
+ ple.com
+ATTENDEE;CUTYPE=INDIVIDUAL;RSVP=TRUE;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-A
+ CTION;CN=nonexistant at example.com:mailto:nonexistant at example.com
+CREATED:20080812T191857Z
+DTSTAMP:20080812T191932Z
+ORGANIZER;CN=User 01:mailto:xyzzy+%s at example.com
+SEQUENCE:2
+SUMMARY:New Event
+TRANSP:OPAQUE
+END:VEVENT
+END:VCALENDAR
+"""
+
+        # Make sure an unknown token is not processed
+        calBody = template % "bogus_token"
+        self.assertEquals(
+            (yield self.receiver.processDSN(calBody, "xyzzy")),
+            MailReceiver.UNKNOWN_TOKEN
+        )
+
+        # Make sure a known token *is* processed
+        txn = self.store.newTransaction()
+        token = (yield txn.imipCreateToken(
+            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
+            "mailto:user02 at example.com",
+            "1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C"
+        ))
+        yield txn.commit()
+        calBody = template % token
+        result = (yield self.receiver.processDSN(calBody, "xyzzy"))
+        self.assertEquals(result, MailReceiver.INJECTION_SUBMITTED)
+
+
+    @inlineCallbacks
+    def test_processReply(self):
+        msg = email.message_from_string(self.dataFile('good_reply'))
+
+        # Make sure an unknown token is not processed
+        result = (yield self.receiver.processReply(msg))
+        self.assertEquals(result, MailReceiver.UNKNOWN_TOKEN)
+
+        # Make sure a known token *is* processed
+        txn = self.store.newTransaction()
+        yield txn.imipCreateToken(
+            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
+            "mailto:xyzzy at example.com",
+            "1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
+            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
+        )
+        yield txn.commit()
+
+        result = (yield self.receiver.processReply(msg))
+        self.assertEquals(result, MailReceiver.INJECTION_SUBMITTED)
+
+
+    def test_processReplyMissingOrganizer(self):
+        msg = email.message_from_string(self.dataFile('reply_missing_organizer'))
+
+        # stick the token in the database first
+        txn = self.store.newTransaction()
+        yield txn.imipCreateToken(
+            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
+            "mailto:xyzzy at example.com",
+            "1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
+            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
+        )
+        yield txn.commit()
+
+        result = (yield self.receiver.processReply(msg))
+        organizer, _ignore_attendee, calendar = result
+        organizerProp = calendar.mainComponent().getOrganizerProperty()
+        self.assertTrue(organizerProp is not None)
+        self.assertEquals(organizer,
+                          "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500")
+
+
+    def test_processReplyMissingAttendee(self):
+        msg = email.message_from_string(self.dataFile('reply_missing_attendee'))
+
+        txn = self.store.newTransaction()
+        yield txn.imipCreateToken(
+            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
+            "mailto:xyzzy at example.com",
+            "1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
+            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
+        )
+        yield txn.commit()
+
+        result = (yield self.receiver.processReply(msg))
+        _ignore_organizer, attendee, calendar = result
+
+        # Since the expected attendee was missing, the reply processor should
+        # have added an attendee back in with a "5.1;Service unavailable"
+        # schedule-status
+        attendeeProp = calendar.mainComponent().getAttendeeProperty([attendee])
+        self.assertEquals(attendeeProp.parameterValue("SCHEDULE-STATUS"),
+                          iTIPRequestStatus.SERVICE_UNAVAILABLE)
+
+
+    def test_processReplyMissingAttachment(self):
+
+        msg = email.message_from_string(
+            self.dataFile('reply_missing_attachment')
+        )
+
+        # stick the token in the database first
+        txn = self.store.newTransaction()
+        yield txn.imipCreateToken(
+            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
+            "mailto:xyzzy at example.com",
+            "1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
+            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
+        )
+        yield txn.commit()
+
+        result = (yield self.receiver.processReply(msg))
+        self.assertEquals(result, MailReceiver.REPLY_FORWARDED_TO_ORGANIZER)        
+
+    @inlineCallbacks
+    def test_injectMessage(self):
+
+        calendar = Component.fromString("""BEGIN:VCALENDAR
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+VERSION:2.0
+METHOD:REPLY
+BEGIN:VEVENT
+UID:12345-67890
+DTSTAMP:20130208T120000Z
+DTSTART:20180601T120000Z
+DTEND:20180601T130000Z
+ORGANIZER:urn:uuid:user01
+ATTENDEE:mailto:xyzzy at example.com;PARTSTAT=ACCEPTED
+END:VEVENT
+END:VCALENDAR
+""")
+
+        txn = self.store.newTransaction()
+        result = (yield injectMessage(
+                txn,
+                self.root,
+                "urn:uuid:user01",
+                "mailto:xyzzy at example.com",
+                calendar
+            )
+        )
+        yield txn.commit()
+        self.assertEquals(
+            "1.2;Scheduling message has been delivered",
+            result.responses[0].children[1].toString()
+        )
+
+    @inlineCallbacks
+    def test_injectMessageWithError(self):
+
+        calendar = Component.fromString("""BEGIN:VCALENDAR
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+VERSION:2.0
+METHOD:REPLY
+BEGIN:VEVENT
+UID:12345-67890
+DTSTAMP:20130208T120000Z
+DTSTART:20180601T120000Z
+DTEND:20180601T130000Z
+ORGANIZER:urn:uuid:unknown_user
+ATTENDEE:mailto:xyzzy at example.com;PARTSTAT=ACCEPTED
+END:VEVENT
+END:VCALENDAR
+""")
+
+        txn = self.store.newTransaction()
+        result = (yield injectMessage(
+                txn,
+                self.root,
+                "urn:uuid:unknown_user",
+                "mailto:xyzzy at example.com",
+                calendar
+            )
+        )
+        yield txn.commit()
+        self.assertEquals(
+            "3.7;Invalid Calendar User",
+            result.responses[0].children[1].toString()
+        )
+
+
+    @inlineCallbacks
+    def test_work(self):
+
+        calendar = """BEGIN:VCALENDAR
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+VERSION:2.0
+METHOD:REPLY
+BEGIN:VEVENT
+UID:12345-67890
+DTSTAMP:20130208T120000Z
+DTSTART:20180601T120000Z
+DTEND:20180601T130000Z
+ORGANIZER:urn:uuid:user01
+ATTENDEE:mailto:xyzzy at example.com;PARTSTAT=ACCEPTED
+END:VEVENT
+END:VCALENDAR
+"""
+        txn = self.store.newTransaction()
+        wp = (yield txn.enqueue(IMIPReplyWork,
+            organizer="urn:uuid:user01",
+            attendee="mailto:xyzzy at example.com",
+            icalendarText=calendar
+        ))
+        yield txn.commit()
+        yield wp.whenExecuted()

Modified: CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_mailgateway.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_mailgateway.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_mailgateway.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -14,858 +14,42 @@
 # limitations under the License.
 ##
 
+import os
 
-from cStringIO import StringIO
-
-from pycalendar.datetime import PyCalendarDateTime
-
 from twisted.internet.defer import inlineCallbacks
-from twisted.python.modules import getModule
-from twisted.web.template import Element, renderer, flattenString
-
-from twistedcaldav.config import config, ConfigDict
-from twistedcaldav.directory import augment
-from twistedcaldav.directory.xmlfile import XMLDirectoryService
-from twistedcaldav.ical import Component
-from twistedcaldav.scheduling.imip.mailgateway import MailGatewayTokensDatabase
-from twistedcaldav.scheduling.imip.mailgateway import MailHandler
-from twistedcaldav.scheduling.imip.mailgateway import StringFormatTemplateLoader
-from twistedcaldav.scheduling.imip.mailgateway import injectionSettingsFromURL
-from twistedcaldav.scheduling.imip.mailgateway import serverForOrganizer
-from twistedcaldav.scheduling.ischedule.localservers import Servers
-from twistedcaldav.scheduling.itip import iTIPRequestStatus
 from twistedcaldav.test.util import TestCase
-from twistedcaldav.test.util import xmlFile, augmentsFile
+from twistedcaldav.test.util import xmlFile
+from txdav.common.datastore.test.util import buildStore
+from calendarserver.tap.util import getRootResource
+from twistedcaldav.config import config
+from twistedcaldav.scheduling.imip.mailgateway import MailGatewayTokensDatabase
+from twistedcaldav.scheduling.imip.mailgateway import migrateTokensToStore
 
-import datetime
-import email
-import os
 
+class MailGatewayTokenDBTests(TestCase):
 
-def echo(*args):
-    return args
-
-initialInviteText = u"""BEGIN:VCALENDAR
-VERSION:2.0
-METHOD:REQUEST
-BEGIN:VEVENT
-UID:CFDD5E46-4F74-478A-9311-B3FF905449C3
-DTSTART:20100325T154500Z
-DTEND:20100325T164500Z
-ATTENDEE;CN=Th\xe9 Attendee;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=TRU
- E:mailto:attendee at example.com
-ATTENDEE;CN=Th\xe9 Organizer;CUTYPE=INDIVIDUAL;EMAIL=organizer at example.com;P
- ARTSTAT=ACCEPTED:urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A
-ATTENDEE;CN=An Attendee without CUTYPE;EMAIL=nocutype at example.com;PARTSTAT=A
- CCEPTED:urn:uuid:4DB528DC-3E60-44FA-9546-2A00FCDCFFAB
-ATTENDEE;EMAIL=nocn at example.com;PARTSTAT=ACCEPTED:urn:uuid:A592CF8B-4FC8-4E4
- F-B543-B2F29A7EEB0B
-ORGANIZER;CN=Th\xe9 Organizer;EMAIL=organizer at example.com:urn:uuid:C3B38B00-
- 4166-11DD-B22C-A07C87E02F6A
-SUMMARY:t\xe9sting outbound( )
-DESCRIPTION:awesome description with "<" and "&"
-END:VEVENT
-END:VCALENDAR
-"""
-
-class MailHandlerTests(TestCase):
-
+    @inlineCallbacks
     def setUp(self):
-        super(MailHandlerTests, self).setUp()
+        super(MailGatewayTokenDBTests, self).setUp()
 
-        self._setupServers(serverData)
-        self.directory = XMLDirectoryService(
-            {
-                'xmlFile' : xmlFile,
-                'augmentService' :
-                    augment.AugmentXMLDB(xmlFiles=(augmentsFile.path,)),
-            }
-        )
-        self.handler = MailHandler(dataRoot=":memory:", directory=self.directory)
-        module = getModule(__name__)
-        self.dataPath = module.filePath.sibling("data")
+        self.store = yield buildStore(self, None)
+        self.patch(config.DirectoryService.params, "xmlFile", xmlFile)
+        self.root = getRootResource(config, self.store)
+        self.directory = self.root.getDirectory()
 
-
-    def _setupServers(self, data):
-        self.patch(config, "ServerHostName", "caldav1.example.com")
-        self.patch(config, "HTTPPort", 8008)
-        self.patch(config.Servers, "Enabled", True)
-
-        xmlFile = StringIO(data)
-        servers = Servers
-        servers.load(xmlFile, ignoreIPLookupFailures=True)
-
-
-    def dataFile(self, name):
-        """
-        Get the contents of a given data file from the 'data/mail' test
-        fixtures directory.
-        """
-        return self.dataPath.child(name).getContent()
-
-
-    def test_serverDetection(self):
-        wsanchez = self.directory.recordWithShortName("users",
-            "wsanchez")
-        cdaboo = self.directory.recordWithShortName("users",
-            "cdaboo")
-        server = wsanchez.server()
-        self.assertEquals(server.uri, "http://caldav1.example.com:8008")
-        server = cdaboo.server()
-        self.assertEquals(server.uri, "https://caldav2.example.com:8843")
-
-        url = serverForOrganizer(self.directory,
-            "mailto:wsanchez at example.com")
-        self.assertEquals(url, "http://caldav1.example.com:8008")
-        url = serverForOrganizer(self.directory,
-            "mailto:cdaboo at example.com")
-        self.assertEquals(url, "https://caldav2.example.com:8843")
-
-
-    def test_purge_and_lowercase(self):
-        """
-        Ensure that purge( ) cleans out old tokens, and that lowercase( )
-        converts all mailto: to lowercase, since earlier server versions
-        didn't do that before inserting into the database.
-        """
-
-        # Insert an "old" token
-        token = "test_token_1"
-        organizer = "urn:uuid:19BFE23D-0269-46CA-877C-D4B521A7A9A5"
-        attendee = "mailto:you at example.com"
-        icaluid = "123"
-        pastDate = datetime.date(2009, 1, 1)
-        self.handler.db._db_execute(
-            """
-            insert into TOKENS (TOKEN, ORGANIZER, ATTENDEE, ICALUID, DATESTAMP)
-            values (:1, :2, :3, :4, :5)
-            """, token, organizer, attendee, icaluid, pastDate
-        )
-        self.handler.db._db_commit()
-
-        # purge, and make sure we don't see that token anymore
-        self.handler.purge()
-        retrieved = self.handler.db.getToken(organizer, attendee, icaluid)
-        self.assertEquals(retrieved, None)
-
-        # Insert a token with (old-format) mailto:
-        token = "test_token_2"
-        organizer = "MailTo:Organizer at Example.com"
-        attendee = "MAILTO:YouTwo at Example.com"
-        icaluid = "456"
-        futureDate = datetime.date(2100, 1, 1)
-        self.handler.db._db_execute(
-            """
-            insert into TOKENS (TOKEN, ORGANIZER, ATTENDEE, ICALUID, DATESTAMP)
-            values (:1, :2, :3, :4, :5)
-            """, token, organizer, attendee, icaluid, futureDate
-        )
-        self.handler.db._db_commit()
-
-        self.handler.lowercase()
-        retrieved = self.handler.db.getToken(organizer.lower(),
-            attendee.lower(), icaluid)
-        self.assertIsInstance(retrieved, str)
-        self.assertEquals(retrieved, token)
-
-        # Insert a token with (new-format) urn:uuid:
-        token = "test_token_3"
-        organizer = "urn:uuid:E0CF4031-676B-4668-A9D3-8F33A0212F70"
-        attendee = "MAILTO:YouTwo at Example.com"
-        icaluid = "789"
-        futureDate = datetime.date(2100, 1, 1)
-        self.handler.db._db_execute(
-            """
-            insert into TOKENS (TOKEN, ORGANIZER, ATTENDEE, ICALUID, DATESTAMP)
-            values (:1, :2, :3, :4, :5)
-            """, token, organizer, attendee, icaluid, futureDate
-        )
-        self.handler.db._db_commit()
-
-        self.handler.lowercase()
-        retrieved = self.handler.db.getToken(organizer,
-            attendee.lower(), icaluid)
-        self.assertEquals(retrieved, token)
-
-
-    def test_checkDSNFailure(self):
-
-        data = {
-            'good_reply' : (False, None, None),
-            'dsn_failure_no_original' : (True, 'failed', None),
-            'dsn_failure_no_ics' : (True, 'failed', None),
-            'dsn_failure_with_ics' : (True, 'failed', '''BEGIN:VCALENDAR
-VERSION:2.0
-CALSCALE:GREGORIAN
-METHOD:REQUEST
-PRODID:-//example Inc.//iCal 3.0//EN
-BEGIN:VTIMEZONE
-TZID:US/Pacific
-BEGIN:STANDARD
-DTSTART:20071104T020000
-RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
-TZNAME:PST
-TZOFFSETFROM:-0700
-TZOFFSETTO:-0800
-END:STANDARD
-BEGIN:DAYLIGHT
-DTSTART:20070311T020000
-RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
-TZNAME:PDT
-TZOFFSETFROM:-0800
-TZOFFSETTO:-0700
-END:DAYLIGHT
-END:VTIMEZONE
-BEGIN:VEVENT
-UID:1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C
-DTSTART;TZID=US/Pacific:20080812T094500
-DTEND;TZID=US/Pacific:20080812T104500
-ATTENDEE;CUTYPE=INDIVIDUAL;CN=User 01;PARTSTAT=ACCEPTED:mailto:user01 at exam
- ple.com
-ATTENDEE;CUTYPE=INDIVIDUAL;RSVP=TRUE;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-A
- CTION;CN=nonexistant at example.com:mailto:nonexistant at example.com
-CREATED:20080812T191857Z
-DTSTAMP:20080812T191932Z
-ORGANIZER;CN=User 01:mailto:xyzzy+8e16b897-d544-4217-88e9-a363d08
- 46f6c at example.com
-SEQUENCE:2
-SUMMARY:New Event
-TRANSP:OPAQUE
-END:VEVENT
-END:VCALENDAR
-'''),
-        }
-
-        for filename, expected in data.iteritems():
-            msg = email.message_from_string(self.dataFile(filename))
-            self.assertEquals(self.handler.checkDSN(msg), expected)
-
-
-    def test_processDSN(self):
-
-        template = """BEGIN:VCALENDAR
-VERSION:2.0
-CALSCALE:GREGORIAN
-METHOD:REQUEST
-PRODID:-//example Inc.//iCal 3.0//EN
-BEGIN:VTIMEZONE
-TZID:US/Pacific
-BEGIN:DAYLIGHT
-DTSTART:20070311T020000
-RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
-TZNAME:PDT
-TZOFFSETFROM:-0800
-TZOFFSETTO:-0700
-END:DAYLIGHT
-BEGIN:STANDARD
-DTSTART:20071104T020000
-RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
-TZNAME:PST
-TZOFFSETFROM:-0700
-TZOFFSETTO:-0800
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-UID:1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C
-DTSTART;TZID=US/Pacific:20080812T094500
-DTEND;TZID=US/Pacific:20080812T104500
-ATTENDEE;CUTYPE=INDIVIDUAL;CN=User 01;PARTSTAT=ACCEPTED:mailto:user01 at exam
- ple.com
-ATTENDEE;CUTYPE=INDIVIDUAL;RSVP=TRUE;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-A
- CTION;CN=nonexistant at example.com:mailto:nonexistant at example.com
-CREATED:20080812T191857Z
-DTSTAMP:20080812T191932Z
-ORGANIZER;CN=User 01:mailto:xyzzy+%s at example.com
-SEQUENCE:2
-SUMMARY:New Event
-TRANSP:OPAQUE
-END:VEVENT
-END:VCALENDAR
-"""
-
-        # Make sure an unknown token is not processed
-        calBody = template % "bogus_token"
-        self.assertEquals(self.handler.processDSN(calBody, "xyzzy", echo),
-           None)
-
-        # Make sure a known token *is* processed
-        token = self.handler.db.createToken(
-            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
-            "mailto:user02 at example.com", "1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C")
-        calBody = template % token
-        _ignore_url, organizer, attendee, calendar, msgId = self.handler.processDSN(calBody,
-            "xyzzy", echo)
-        self.assertEquals(organizer, 'urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500')
-        self.assertEquals(attendee, 'mailto:user02 at example.com')
-        self.assertEquals(str(calendar), """BEGIN:VCALENDAR
-VERSION:2.0
-CALSCALE:GREGORIAN
-METHOD:REQUEST
-PRODID:-//example Inc.//iCal 3.0//EN
-BEGIN:VTIMEZONE
-TZID:US/Pacific
-BEGIN:DAYLIGHT
-DTSTART:20070311T020000
-RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3
-TZNAME:PDT
-TZOFFSETFROM:-0800
-TZOFFSETTO:-0700
-END:DAYLIGHT
-BEGIN:STANDARD
-DTSTART:20071104T020000
-RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11
-TZNAME:PST
-TZOFFSETFROM:-0700
-TZOFFSETTO:-0800
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-UID:1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C
-DTSTART;TZID=US/Pacific:20080812T094500
-DTEND;TZID=US/Pacific:20080812T104500
-CREATED:20080812T191857Z
-DTSTAMP:20080812T191932Z
-ORGANIZER;CN=User 01:urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500
-REQUEST-STATUS:5.1;Service unavailable
-SEQUENCE:2
-SUMMARY:New Event
-TRANSP:OPAQUE
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-        self.assertEquals(msgId, 'xyzzy')
-
-
-    def test_processReply(self):
-        msg = email.message_from_string(self.dataFile('good_reply'))
-
-        # Make sure an unknown token is not processed
-        result = self.handler.processReply(msg, echo)
-        self.assertEquals(result, None)
-
-        # Make sure a known token *is* processed
-        self.handler.db.createToken(
-            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
-            "mailto:xyzzy at example.com",
-            icaluid="1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
-            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
-        )
-        url, organizer, attendee, _ignore_calendar, msgId = self.handler.processReply(msg, echo)
-        self.assertEquals(url, "https://caldav2.example.com:8843")
-        self.assertEquals(organizer,
-                          'urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500')
-        self.assertEquals(attendee, 'mailto:xyzzy at example.com')
-        self.assertEquals(msgId,
-                          '<1983F777-BE86-4B98-881E-06D938E60920 at example.com>')
-
-
-    def test_injectionSettingsFromURL(self):
-        testData = (
-            (
-                None,
-                {
-                    "Scheduling": {
-                        "iMIP" : {
-                            "MailGatewayServer" : "localhost",
-                        },
-                    },
-                    "EnableSSL" : True,
-                    "ServerHostName" : "calendar.example.com",
-                    "HTTPPort" : 1111,
-                    "SSLPort" : 2222,
-                },
-                "https://localhost:2222/inbox/",
-            ),
-            (
-                None,
-                {
-                    "Scheduling": {
-                        "iMIP" : {
-                            "MailGatewayServer" : "mailgateway.example.com",
-                        },
-                    },
-                    "EnableSSL" : False,
-                    "ServerHostName" : "calendar.example.com",
-                    "HTTPPort" : 1111,
-                    "SSLPort" : 2222,
-                },
-                "http://calendar.example.com:1111/inbox/",
-            ),
-            (
-                "https://calendar.example.com:1234/",
-                { },
-                "https://calendar.example.com:1234/inbox/",
-            ),
-            (
-                "https://calendar.example.com:1234",
-                { },
-                "https://calendar.example.com:1234/inbox/",
-            ),
-        )
-
-        for url, configData, expected in testData:
-            self.assertEquals(
-                expected,
-                injectionSettingsFromURL(url, ConfigDict(mapping=configData))
-            )
-
-
-    def test_processReplyMissingOrganizer(self):
-        msg = email.message_from_string(self.dataFile('reply_missing_organizer'))
-        # stick the token in the database first
-        self.handler.db.createToken(
-            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
-            "mailto:xyzzy at example.com",
-            icaluid="1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
-            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
-        )
-
-        _ignore_url, organizer, _ignore_attendee, calendar, _ignore_msgId = self.handler.processReply(
-            msg, echo)
-        organizerProp = calendar.mainComponent().getOrganizerProperty()
-        self.assertTrue(organizerProp is not None)
-        self.assertEquals(organizer,
-                          "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500")
-
-
-    def test_processReplyMissingAttendee(self):
-        msg = email.message_from_string(self.dataFile('reply_missing_attendee'))
-
-        # stick the token in the database first
-        self.handler.db.createToken(
-            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
-            "mailto:xyzzy at example.com",
-            icaluid="1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
-            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
-        )
-
-        _ignore_url, _ignore_organizer, attendee, calendar, _ignore_msgId = self.handler.processReply(
-            msg, echo)
-
-        # Since the expected attendee was missing, the reply processor should
-        # have added an attendee back in with a "5.1;Service unavailable"
-        # schedule-status
-        attendeeProp = calendar.mainComponent().getAttendeeProperty([attendee])
-        self.assertEquals(attendeeProp.parameterValue("SCHEDULE-STATUS"),
-                          iTIPRequestStatus.SERVICE_UNAVAILABLE)
-
-
-    def test_processReplyMissingAttachment(self):
-
-        msg = email.message_from_string(
-            self.dataFile('reply_missing_attachment')
-        )
-        # stick the token in the database first
-        self.handler.db.createToken(
-            "urn:uuid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
-            "mailto:xyzzy at example.com",
-            icaluid="1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
-            token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
-        )
-
-        self.assertEquals(
-            self.handler.processReply(msg, echo, testMode=True),
-            ("cdaboo at example.com", "xyzzy at example.com")
-        )
-
-
     @inlineCallbacks
-    def test_outbound(self):
-        """
-        Make sure outbound( ) stores tokens properly so they can be looked up
-        """
+    def test_migrate(self):
+        self.path = self.mktemp()
+        os.mkdir(self.path)
+        oldDB = MailGatewayTokensDatabase(self.path)
+        oldDB.createToken("urn:uuid:user01", "mailto:attendee at example.com",
+            "icaluid1", token="token1")
+        yield migrateTokensToStore(self.path, self.store)
+        txn = self.store.newTransaction()
+        results = yield (txn.imipLookupByToken("token1"))
+        organizer, attendee, icaluid = results[0]
+        yield txn.commit()
+        self.assertEquals(organizer, "urn:uuid:user01")
+        self.assertEquals(attendee, "mailto:attendee at example.com")
+        self.assertEquals(icaluid, "icaluid1")
 
-        config.Scheduling.iMIP.Sending.Address = "server at example.com"
-        self.patch(config.Localization, "LocalesDirectory", os.path.join(os.path.dirname(__file__), "locales"))
-
-        data = (
-            # Initial invite
-            (
-                initialInviteText,
-                "CFDD5E46-4F74-478A-9311-B3FF905449C3",
-                "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A",
-                "mailto:attendee at example.com",
-                "new",
-                "organizer at example.com",
-                u"Th\xe9 Organizer",
-                [
-                    (u'Th\xe9 Attendee', u'attendee at example.com'),
-                    (u'Th\xe9 Organizer', u'organizer at example.com'),
-                    (u'An Attendee without CUTYPE', u'nocutype at example.com'),
-                    (None, u'nocn at example.com'),
-                ],
-                u"Th\xe9 Organizer <organizer at example.com>",
-                "attendee at example.com",
-            ),
-
-            # Update
-            (
-                """BEGIN:VCALENDAR
-VERSION:2.0
-METHOD:REQUEST
-BEGIN:VEVENT
-UID:CFDD5E46-4F74-478A-9311-B3FF905449C3
-DTSTART:20100325T154500Z
-DTEND:20100325T164500Z
-ATTENDEE;CN=The Attendee;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:
- mailto:attendee at example.com
-ATTENDEE;CN=The Organizer;CUTYPE=INDIVIDUAL;EMAIL=organizer at example.com;PAR
- TSTAT=ACCEPTED:urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A
-ORGANIZER;CN=The Organizer;EMAIL=organizer at example.com:urn:uuid:C3B38B00-41
- 66-11DD-B22C-A07C87E02F6A
-SUMMARY:testing outbound( ) *update*
-END:VEVENT
-END:VCALENDAR
-""",
-                "CFDD5E46-4F74-478A-9311-B3FF905449C3",
-                "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A",
-                "mailto:attendee at example.com",
-                "update",
-                "organizer at example.com",
-                "The Organizer",
-                [
-                    (u'The Attendee', u'attendee at example.com'),
-                    (u'The Organizer', u'organizer at example.com')
-                ],
-                "The Organizer <organizer at example.com>",
-                "attendee at example.com",
-            ),
-
-            # Reply
-            (
-                """BEGIN:VCALENDAR
-VERSION:2.0
-METHOD:REPLY
-BEGIN:VEVENT
-UID:DFDD5E46-4F74-478A-9311-B3FF905449C4
-DTSTART:20100325T154500Z
-DTEND:20100325T164500Z
-ATTENDEE;CN=The Attendee;CUTYPE=INDIVIDUAL;EMAIL=attendee at example.com;PARTST
- AT=ACCEPTED:urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A
-ORGANIZER;CN=The Organizer;EMAIL=organizer at example.com:mailto:organizer at exam
- ple.com
-SUMMARY:testing outbound( ) *reply*
-END:VEVENT
-END:VCALENDAR
-""",
-                None,
-                "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A",
-                "mailto:organizer at example.com",
-                "reply",
-                "organizer at example.com",
-                "The Organizer",
-                [
-                    (u'The Attendee', u'attendee at example.com'),
-                ],
-                "attendee at example.com",
-                "organizer at example.com",
-            ),
-
-        )
-        for (inputCalendar, UID, inputOriginator, inputRecipient, inviteState,
-            outputOrganizerEmail, outputOrganizerName, outputAttendeeList,
-            outputFrom, outputRecipient) in data:
-
-            (actualInviteState, actualCalendar, actualOrganizerEmail,
-                actualOrganizerName, actualAttendeeList, actualFrom,
-                actualRecipient, actualReplyTo) = (yield self.handler.outbound(
-                    inputOriginator,
-                    inputRecipient,
-                    Component.fromString(inputCalendar.replace("\n", "\r\n")),
-                    language="ja",
-                    send=False,
-                    onlyAfter=PyCalendarDateTime(2010, 1, 1, 0, 0, 0))
-                )
-
-            self.assertEquals(actualInviteState, inviteState)
-            self.assertEquals(actualOrganizerEmail, outputOrganizerEmail)
-            self.assertEquals(actualOrganizerName, outputOrganizerName)
-            self.assertEquals(actualAttendeeList, outputAttendeeList)
-            self.assertEquals(actualFrom, outputFrom)
-            self.assertEquals(actualRecipient, outputRecipient)
-
-            if UID: # The organizer is local, and server is sending to remote
-                    # attendee
-
-                token = self.handler.db.getToken(inputOriginator,
-                    inputRecipient, UID)
-                self.assertNotEquals(token, None)
-                self.assertEquals(actualReplyTo,
-                    "server+%s at example.com" % (token,))
-
-                # Make sure attendee property for organizer exists and matches
-                # the CUA of the organizer property
-                orgValue = actualCalendar.getOrganizerProperty().value()
-                self.assertEquals(
-                    orgValue,
-                    actualCalendar.getAttendeeProperty([orgValue]).value()
-                )
-
-            else: # Reply only -- the attendee is local, and server is sending reply to remote organizer
-
-                self.assertEquals(actualReplyTo, actualFrom)
-
-            # Check that we don't send any messages for events completely in
-            # the past.
-            result = (yield self.handler.outbound(
-                    inputOriginator,
-                    inputRecipient,
-                    Component.fromString(inputCalendar.replace("\n", "\r\n")),
-                    send=False,
-                    onlyAfter=PyCalendarDateTime(2012, 1, 1, 0, 0, 0))
-                )
-            self.assertEquals(result, True)
-
-
-    @inlineCallbacks
-    def test_mailtoTokens(self):
-        """
-        Make sure old mailto tokens are still honored
-        """
-
-        organizerEmail = "mailto:organizer at example.com"
-
-        config.Scheduling.iMIP.Sending.Address = "server at example.com"
-
-        # Explictly store a token with mailto: CUA for organizer
-        # (something that doesn't happen any more, but did in the past)
-        origToken = self.handler.db.createToken(organizerEmail,
-            "mailto:attendee at example.com",
-            "CFDD5E46-4F74-478A-9311-B3FF905449C3")
-
-        inputCalendar = initialInviteText
-        UID = "CFDD5E46-4F74-478A-9311-B3FF905449C3"
-        inputOriginator = "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A"
-        inputRecipient = "mailto:attendee at example.com"
-
-        (_ignore_actualInviteState, _ignore_actualCalendar, _ignore_actualOrganizerEmail,
-            _ignore_actualOrganizerName, _ignore_actualAttendeeList, _ignore_actualFrom,
-            _ignore_actualRecipient, _ignore_actualReplyTo) = (yield self.handler.outbound(
-                inputOriginator,
-                inputRecipient,
-                Component.fromString(inputCalendar.replace("\n", "\r\n")),
-                send=False,
-                onlyAfter=PyCalendarDateTime(2010, 1, 1, 0, 0, 0))
-            )
-
-        # Verify we didn't create a new token...
-        token = self.handler.db.getToken(inputOriginator,
-            inputRecipient, UID)
-        self.assertEquals(token, None)
-
-        # But instead kept the old one...
-        token = self.handler.db.getToken(organizerEmail,
-            inputRecipient, UID)
-        self.assertEquals(token, origToken)
-
-
-    def generateSampleEmail(self):
-        """
-        Invoke L{MailHandler.generateEmail} and parse the result.
-        """
-        calendar = Component.fromString(initialInviteText)
-        msgID, msgTxt = self.handler.generateEmail(
-            inviteState='new',
-            calendar=calendar,
-            orgEmail=u"user01 at localhost",
-            orgCN=u"User Z\xe9ro One",
-            attendees=[(u"Us\xe9r One", "user01 at localhost"),
-                       (u"User 2", "user02 at localhost")],
-            fromAddress="user01 at localhost",
-            replyToAddress="imip-system at localhost",
-            toAddress="user03 at localhost",
-        )
-        message = email.message_from_string(msgTxt)
-        return msgID, message
-
-
-    def test_generateEmail(self):
-        """
-        L{MailHandler.generateEmail} generates a MIME-formatted email with a
-        text/plain part, a text/html part, and a text/calendar part.
-        """
-        msgID, message = self.generateSampleEmail()
-        self.assertEquals(message['Message-ID'], msgID)
-        expectedTypes = set(["text/plain", "text/html", "text/calendar"])
-        actualTypes = set([
-            part.get_content_type() for part in message.walk()
-            if part.get_content_type().startswith("text/")
-        ])
-        self.assertEquals(actualTypes, expectedTypes)
-
-
-    def test_emailEncoding(self):
-        """
-        L{MailHandler.generateEmail} will preserve any non-ASCII characters
-        present in the fields that it formats in the message body.
-        """
-        _ignore_msgID, message = self.generateSampleEmail()
-        textPart = partByType(message, "text/plain")
-        htmlPart = partByType(message, "text/html")
-
-        plainText = textPart.get_payload(decode=True).decode(
-            textPart.get_content_charset()
-        )
-        htmlText = htmlPart.get_payload(decode=True).decode(
-            htmlPart.get_content_charset()
-        )
-
-        self.assertIn(u"Us\u00e9r One", plainText)
-        self.assertIn(u'<a href="mailto:user01 at localhost">Us\u00e9r One</a>',
-                      htmlText)
-
-        # The same assertion, but with the organizer's form.
-        self.assertIn(
-            u'<a href="mailto:user01 at localhost">User Z\u00e9ro One</a>',
-            htmlText)
-
-
-    def test_emailQuoting(self):
-        """
-        L{MailHandler.generateEmail} will HTML-quote all relevant fields in the
-        HTML part, but not the text/plain part.
-        """
-        _ignore_msgID, message = self.generateSampleEmail()
-        htmlPart = partByType(message, "text/html").get_payload(decode=True)
-        plainPart = partByType(message, "text/plain").get_payload(decode=True)
-        expectedPlain = 'awesome description with "<" and "&"'
-        expectedHTML = expectedPlain.replace("&", "&amp;").replace("<", "&lt;")
-
-        self.assertIn(expectedPlain, plainPart)
-        self.assertIn(expectedHTML, htmlPart)
-
-
-    def test_stringFormatTemplateLoader(self):
-        """
-        L{StringFormatTemplateLoader.load} will convert a template with
-        C{%(x)s}-format slots by converting it to a template with C{<t:slot
-        name="x" />} slots, and a renderer on the document element named
-        according to the constructor argument.
-        """
-        class StubElement(Element):
-            loader = StringFormatTemplateLoader(
-                lambda : StringIO(
-                    "<test><alpha>%(slot1)s</alpha>%(other)s</test>"
-                ),
-                "testRenderHere"
-            )
-
-            @renderer
-            def testRenderHere(self, request, tag):
-                return tag.fillSlots(slot1="hello",
-                                     other="world")
-        result = []
-        flattenString(None, StubElement()).addCallback(result.append)
-        self.assertEquals(result,
-                          ["<test><alpha>hello</alpha>world</test>"])
-
-
-    def test_templateLoaderWithAttributes(self):
-        """
-        L{StringFormatTemplateLoader.load} will convert a template with
-        C{%(x)s}-format slots inside attributes into t:attr elements containing
-        t:slot slots.
-        """
-        class StubElement(Element):
-            loader = StringFormatTemplateLoader(
-                lambda : StringIO(
-                    '<test><alpha beta="before %(slot1)s after">inner</alpha>'
-                    '%(other)s</test>'
-                ),
-                "testRenderHere"
-            )
-
-            @renderer
-            def testRenderHere(self, request, tag):
-                return tag.fillSlots(slot1="hello",
-                                     other="world")
-        result = []
-        flattenString(None, StubElement()).addCallback(result.append)
-        self.assertEquals(result,
-                          ['<test><alpha beta="before hello after">'
-                           'inner</alpha>world</test>'])
-
-
-    def test_templateLoaderTagSoup(self):
-        """
-        L{StringFormatTemplateLoader.load} will convert a template with
-        C{%(x)s}-format slots into t:slot slots, and render a well-formed output
-        document, even if the input is malformed (i.e. missing necessary closing
-        tags).
-        """
-        class StubElement(Element):
-            loader = StringFormatTemplateLoader(
-                lambda : StringIO(
-                    '<test><alpha beta="before %(slot1)s after">inner</alpha>'
-                    '%(other)s'
-                ),
-                "testRenderHere"
-            )
-
-            @renderer
-            def testRenderHere(self, request, tag):
-                return tag.fillSlots(slot1="hello",
-                                     other="world")
-        result = []
-        flattenString(None, StubElement()).addCallback(result.append)
-        self.assertEquals(result,
-                          ['<test><alpha beta="before hello after">'
-                           'inner</alpha>world</test>'])
-
-
-
-def partByType(message, contentType):
-    """
-    Retrieve a MIME part from an L{email.message.Message} based on a content
-    type.
-    """
-    for part in message.walk():
-        if part.get_content_type() == contentType:
-            return part
-    raise KeyError(contentType)
-
-
-
-class MailGatewayTokensDatabaseTests(TestCase):
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.db = MailGatewayTokensDatabase(":memory:")
-
-
-    def test_tokens(self):
-        self.assertEquals(self.db.lookupByToken("xyzzy"), None)
-
-        token = self.db.createToken("organizer", "attendee", "icaluid")
-        self.assertEquals(self.db.getToken("organizer", "attendee", "icaluid"),
-                          token)
-        self.assertEquals(self.db.lookupByToken(token),
-            ("organizer", "attendee", "icaluid"))
-        self.db.deleteToken(token)
-        self.assertEquals(self.db.lookupByToken(token), None)
-
-
-serverData = """<?xml version="1.0" encoding="utf-8"?>
-<servers>
-  <server>
-    <id>00001</id>
-    <uri>http://caldav1.example.com:8008</uri>
-    <allowed-from>127.0.0.1</allowed-from>
-    <shared-secret>foobar</shared-secret>
-  </server>
-  <server>
-    <id>00002</id>
-    <uri>https://caldav2.example.com:8843</uri>
-    <partitions>
-        <partition>
-            <id>A</id>
-            <uri>https://machine1.example.com:8443</uri>
-        </partition>
-        <partition>
-            <id>B</id>
-            <uri>https://machine2.example.com:8443</uri>
-        </partition>
-    </partitions>
-  </server>
-</servers>
-"""

Added: CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_outbound.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_outbound.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/scheduling/imip/test/test_outbound.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,556 @@
+##
+# Copyright (c) 2008-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from cStringIO import StringIO
+import os
+
+import email
+from pycalendar.datetime import PyCalendarDateTime
+from twisted.internet.defer import inlineCallbacks, succeed
+from twisted.web.template import Element, renderer, flattenString
+from twistedcaldav.config import config
+from twistedcaldav.directory import augment
+from twistedcaldav.directory.xmlfile import XMLDirectoryService
+from twistedcaldav.ical import Component
+from twistedcaldav.scheduling.imip.outbound import IMIPInvitationWork
+from twistedcaldav.scheduling.imip.outbound import MailSender
+from twistedcaldav.scheduling.imip.outbound import StringFormatTemplateLoader
+from twistedcaldav.test.util import TestCase, xmlFile, augmentsFile
+from txdav.common.datastore.test.util import buildStore
+
+initialInviteText = u"""BEGIN:VCALENDAR
+VERSION:2.0
+METHOD:REQUEST
+BEGIN:VEVENT
+UID:CFDD5E46-4F74-478A-9311-B3FF905449C3
+DTSTART:20200325T154500Z
+DTEND:20200325T164500Z
+ATTENDEE;CN=Th\xe9 Attendee;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=TRU
+ E:mailto:attendee at example.com
+ATTENDEE;CN=Th\xe9 Organizer;CUTYPE=INDIVIDUAL;EMAIL=organizer at example.com;P
+ ARTSTAT=ACCEPTED:urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A
+ATTENDEE;CN=An Attendee without CUTYPE;EMAIL=nocutype at example.com;PARTSTAT=A
+ CCEPTED:urn:uuid:4DB528DC-3E60-44FA-9546-2A00FCDCFFAB
+ATTENDEE;EMAIL=nocn at example.com;PARTSTAT=ACCEPTED:urn:uuid:A592CF8B-4FC8-4E4
+ F-B543-B2F29A7EEB0B
+ORGANIZER;CN=Th\xe9 Organizer;EMAIL=organizer at example.com:urn:uuid:C3B38B00-
+ 4166-11DD-B22C-A07C87E02F6A
+SUMMARY:t\xe9sting outbound( )
+DESCRIPTION:awesome description with "<" and "&"
+END:VEVENT
+END:VCALENDAR
+"""
+
+ORGANIZER = "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A"
+ATTENDEE = "mailto:attendee at example.com"
+ICALUID = "CFDD5E46-4F74-478A-9311-B3FF905449C3"
+
+class DummySMTPSender(object):
+
+    def __init__(self):
+        self.reset()
+        self.shouldSucceed = True
+
+    def reset(self):
+        self.sendMessageCalled = False
+        self.fromAddr = None
+        self.toAddr = None
+        self.msgId = None
+        self.message = None
+
+    def sendMessage(self, fromAddr, toAddr, msgId, message):
+        self.sendMessageCalled = True
+        self.fromAddr = fromAddr
+        self.toAddr = toAddr
+        self.msgId = msgId
+        self.message = message
+        return succeed(self.shouldSucceed)
+
+
+class OutboundTests(TestCase):
+
+    @inlineCallbacks
+    def setUp(self):
+        self.store = yield buildStore(self, None)
+        self.directory = XMLDirectoryService(
+            {
+                'xmlFile' : xmlFile,
+                'augmentService' :
+                    augment.AugmentXMLDB(xmlFiles=(augmentsFile.path,)),
+            }
+        )
+        self.sender = MailSender("server at example.com", 7, DummySMTPSender())
+
+        def _getSender(ignored):
+            return self.sender
+        self.patch(IMIPInvitationWork, "getMailSender", _getSender)
+
+        self.wp = None
+        self.store.queuer.callWithNewProposals(self._proposalCallback)
+
+    def _proposalCallback(self, wp):
+        # print "New proposal", wp
+        self.wp = wp
+
+    @inlineCallbacks
+    def test_work(self):
+        txn = self.store.newTransaction()
+        wp = (yield txn.enqueue(IMIPInvitationWork,
+            fromAddr=ORGANIZER,
+            toAddr=ATTENDEE,
+            icalendarText=initialInviteText.replace("\n", "\r\n"),
+        ))
+        self.assertEquals(wp, self.wp)
+        yield txn.commit()
+        yield wp.whenExecuted()
+
+        txn = self.store.newTransaction()
+        token = (yield txn.imipGetToken(
+            ORGANIZER,
+            ATTENDEE,
+            ICALUID
+        ))
+        self.assertTrue(token)
+        organizer, attendee, icaluid = (yield txn.imipLookupByToken(token))[0]
+        yield txn.commit()
+        self.assertEquals(organizer, ORGANIZER)
+        self.assertEquals(attendee, ATTENDEE)
+        self.assertEquals(icaluid, ICALUID)
+
+    @inlineCallbacks
+    def test_workFailure(self):
+        self.sender.smtpSender.shouldSucceed = False
+
+        txn = self.store.newTransaction()
+        wp = (yield txn.enqueue(IMIPInvitationWork,
+            fromAddr=ORGANIZER,
+            toAddr=ATTENDEE,
+            icalendarText=initialInviteText.replace("\n", "\r\n"),
+        ))
+        yield txn.commit()
+        yield wp.whenExecuted()
+        # Verify a new work proposal was not created
+        self.assertEquals(wp, self.wp)
+
+
+    def _interceptEmail(self, inviteState, calendar, orgEmail, orgCn,
+        attendees, fromAddress, replyToAddress, toAddress, language="en"):
+        self.inviteState = inviteState
+        self.calendar = calendar
+        self.orgEmail = orgEmail
+        self.orgCn = orgCn
+        self.attendees = attendees
+        self.fromAddress = fromAddress
+        self.replyToAddress = replyToAddress
+        self.toAddress = toAddress
+        self.language = language
+        self.results = self._actualGenerateEmail(inviteState, calendar,
+            orgEmail, orgCn, attendees, fromAddress, replyToAddress, toAddress,
+            language=language)
+        return self.results
+
+    @inlineCallbacks
+    def test_outbound(self):
+        """
+        Make sure outbound( ) stores tokens properly so they can be looked up
+        """
+
+        config.Scheduling.iMIP.Sending.Address = "server at example.com"
+        self.patch(config.Localization, "LocalesDirectory", os.path.join(os.path.dirname(__file__), "locales"))
+        self._actualGenerateEmail = self.sender.generateEmail
+        self.patch(self.sender, "generateEmail", self._interceptEmail)
+
+        data = (
+            # Initial invite
+            (
+                initialInviteText,
+                "CFDD5E46-4F74-478A-9311-B3FF905449C3",
+                "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A",
+                "mailto:attendee at example.com",
+                "new",
+                "organizer at example.com",
+                u"Th\xe9 Organizer",
+                [
+                    (u'Th\xe9 Attendee', u'attendee at example.com'),
+                    (u'Th\xe9 Organizer', u'organizer at example.com'),
+                    (u'An Attendee without CUTYPE', u'nocutype at example.com'),
+                    (None, u'nocn at example.com'),
+                ],
+                u"Th\xe9 Organizer <organizer at example.com>",
+                "=?utf-8?q?Th=C3=A9_Organizer_=3Corganizer=40example=2Ecom=3E?=",
+                "attendee at example.com",
+            ),
+
+            # Update
+            (
+                """BEGIN:VCALENDAR
+VERSION:2.0
+METHOD:REQUEST
+BEGIN:VEVENT
+UID:CFDD5E46-4F74-478A-9311-B3FF905449C3
+DTSTART:20100325T154500Z
+DTEND:20100325T164500Z
+ATTENDEE;CN=The Attendee;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:
+ mailto:attendee at example.com
+ATTENDEE;CN=The Organizer;CUTYPE=INDIVIDUAL;EMAIL=organizer at example.com;PAR
+ TSTAT=ACCEPTED:urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A
+ORGANIZER;CN=The Organizer;EMAIL=organizer at example.com:urn:uuid:C3B38B00-41
+ 66-11DD-B22C-A07C87E02F6A
+SUMMARY:testing outbound( ) *update*
+END:VEVENT
+END:VCALENDAR
+""",
+                "CFDD5E46-4F74-478A-9311-B3FF905449C3",
+                "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A",
+                "mailto:attendee at example.com",
+                "update",
+                "organizer at example.com",
+                "The Organizer",
+                [
+                    (u'The Attendee', u'attendee at example.com'),
+                    (u'The Organizer', u'organizer at example.com')
+                ],
+                "The Organizer <organizer at example.com>",
+                "The Organizer <organizer at example.com>",
+                "attendee at example.com",
+            ),
+
+            # Reply
+            (
+                """BEGIN:VCALENDAR
+VERSION:2.0
+METHOD:REPLY
+BEGIN:VEVENT
+UID:DFDD5E46-4F74-478A-9311-B3FF905449C4
+DTSTART:20100325T154500Z
+DTEND:20100325T164500Z
+ATTENDEE;CN=The Attendee;CUTYPE=INDIVIDUAL;EMAIL=attendee at example.com;PARTST
+ AT=ACCEPTED:urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A
+ORGANIZER;CN=The Organizer;EMAIL=organizer at example.com:mailto:organizer at exam
+ ple.com
+SUMMARY:testing outbound( ) *reply*
+END:VEVENT
+END:VCALENDAR
+""",
+                None,
+                "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A",
+                "mailto:organizer at example.com",
+                "reply",
+                "organizer at example.com",
+                "The Organizer",
+                [
+                    (u'The Attendee', u'attendee at example.com'),
+                ],
+                "attendee at example.com",
+                "attendee at example.com",
+                "organizer at example.com",
+            ),
+
+        )
+        for (inputCalendar, UID, inputOriginator, inputRecipient, inviteState,
+            outputOrganizerEmail, outputOrganizerName, outputAttendeeList,
+            outputFrom, encodedFrom, outputRecipient) in data:
+
+            txn = self.store.newTransaction()
+            yield self.sender.outbound(
+                txn,
+                inputOriginator,
+                inputRecipient,
+                Component.fromString(inputCalendar.replace("\n", "\r\n")),
+                language="ja",
+                onlyAfter=PyCalendarDateTime(2010, 1, 1, 0, 0, 0)
+            )
+            yield txn.commit()
+
+            msg = email.message_from_string(self.sender.smtpSender.message)
+            self.assertEquals(msg["From"], encodedFrom)
+            self.assertEquals(self.inviteState, inviteState)
+            self.assertEquals(self.orgEmail, outputOrganizerEmail)
+            self.assertEquals(self.orgCn, outputOrganizerName)
+            self.assertEquals(self.attendees, outputAttendeeList)
+            self.assertEquals(self.fromAddress, outputFrom)
+            self.assertEquals(self.toAddress, outputRecipient)
+
+            if UID: # The organizer is local, and server is sending to remote
+                    # attendee
+                txn = self.store.newTransaction()
+                token = (yield txn.imipGetToken(inputOriginator, inputRecipient,
+                    UID))
+                yield txn.commit()
+                self.assertNotEquals(token, None)
+                self.assertEquals(msg["Reply-To"],
+                    "server+%s at example.com" % (token,))
+
+                # Make sure attendee property for organizer exists and matches
+                # the CUA of the organizer property
+                orgValue = self.calendar.getOrganizerProperty().value()
+                self.assertEquals(
+                    orgValue,
+                    self.calendar.getAttendeeProperty([orgValue]).value()
+                )
+
+            else: # Reply only -- the attendee is local, and server is sending reply to remote organizer
+
+                self.assertEquals(msg["Reply-To"], self.fromAddress)
+
+            # Check that we don't send any messages for events completely in
+            # the past.
+            self.sender.smtpSender.reset()
+            txn = self.store.newTransaction()
+            yield self.sender.outbound(
+                txn,
+                inputOriginator,
+                inputRecipient,
+                Component.fromString(inputCalendar.replace("\n", "\r\n")),
+                onlyAfter=PyCalendarDateTime(2021, 1, 1, 0, 0, 0)
+            )
+            yield txn.commit()
+            self.assertFalse(self.sender.smtpSender.sendMessageCalled)
+
+
+    @inlineCallbacks
+    def test_tokens(self):
+        txn = self.store.newTransaction()
+        token = (yield txn.imipLookupByToken("xyzzy"))
+        yield txn.commit()
+        self.assertEquals(token, [])
+
+        txn = self.store.newTransaction()
+        token1 = (yield txn.imipCreateToken("organizer", "attendee", "icaluid"))
+        yield txn.commit()
+
+        txn = self.store.newTransaction()
+        token2 = (yield txn.imipGetToken("organizer", "attendee", "icaluid"))
+        yield txn.commit()
+        self.assertEquals(token1, token2)
+
+        txn = self.store.newTransaction()
+        self.assertEquals((yield txn.imipLookupByToken(token1)),
+            [["organizer", "attendee", "icaluid"]])
+        yield txn.commit()
+
+        txn = self.store.newTransaction()
+        yield txn.imipRemoveToken(token1)
+        yield txn.commit()
+
+        txn = self.store.newTransaction()
+        self.assertEquals((yield txn.imipLookupByToken(token1)), [])
+        yield txn.commit()
+        
+ 
+    @inlineCallbacks
+    def test_mailtoTokens(self):
+        """
+        Make sure old mailto tokens are still honored
+        """
+
+        organizerEmail = "mailto:organizer at example.com"
+
+        # Explictly store a token with mailto: CUA for organizer
+        # (something that doesn't happen any more, but did in the past)
+        txn = self.store.newTransaction()
+        origToken = (yield txn.imipCreateToken(organizerEmail,
+            "mailto:attendee at example.com",
+            "CFDD5E46-4F74-478A-9311-B3FF905449C3"
+            )
+        )
+        yield txn.commit()
+
+        inputCalendar = initialInviteText
+        UID = "CFDD5E46-4F74-478A-9311-B3FF905449C3"
+        inputOriginator = "urn:uuid:C3B38B00-4166-11DD-B22C-A07C87E02F6A"
+        inputRecipient = "mailto:attendee at example.com"
+
+        txn = self.store.newTransaction()
+        yield self.sender.outbound(txn, inputOriginator, inputRecipient,
+            Component.fromString(inputCalendar.replace("\n", "\r\n")),
+            onlyAfter=PyCalendarDateTime(2010, 1, 1, 0, 0, 0))
+
+        # Verify we didn't create a new token...
+        txn = self.store.newTransaction()
+        token = (yield txn.imipGetToken(inputOriginator, inputRecipient, UID))
+        yield txn.commit()
+        self.assertEquals(token, None)
+
+        # But instead kept the old one...
+        txn = self.store.newTransaction()
+        token = (yield txn.imipGetToken(organizerEmail, inputRecipient, UID))
+        yield txn.commit()
+        self.assertEquals(token, origToken)
+
+
+    def generateSampleEmail(self):
+        """
+        Invoke L{MailHandler.generateEmail} and parse the result.
+        """
+        calendar = Component.fromString(initialInviteText)
+        msgID, msgTxt = self.sender.generateEmail(
+            inviteState='new',
+            calendar=calendar,
+            orgEmail=u"user01 at localhost",
+            orgCN=u"User Z\xe9ro One",
+            attendees=[(u"Us\xe9r One", "user01 at localhost"),
+                       (u"User 2", "user02 at localhost")],
+            fromAddress="user01 at localhost",
+            replyToAddress="imip-system at localhost",
+            toAddress="user03 at localhost",
+        )
+        message = email.message_from_string(msgTxt)
+        return msgID, message
+
+
+    def test_generateEmail(self):
+        """
+        L{MailHandler.generateEmail} generates a MIME-formatted email with a
+        text/plain part, a text/html part, and a text/calendar part.
+        """
+        msgID, message = self.generateSampleEmail()
+        self.assertEquals(message['Message-ID'], msgID)
+        expectedTypes = set(["text/plain", "text/html", "text/calendar"])
+        actualTypes = set([
+            part.get_content_type() for part in message.walk()
+            if part.get_content_type().startswith("text/")
+        ])
+        self.assertEquals(actualTypes, expectedTypes)
+
+
+    def test_emailEncoding(self):
+        """
+        L{MailHandler.generateEmail} will preserve any non-ASCII characters
+        present in the fields that it formats in the message body.
+        """
+        _ignore_msgID, message = self.generateSampleEmail()
+        textPart = partByType(message, "text/plain")
+        htmlPart = partByType(message, "text/html")
+
+        plainText = textPart.get_payload(decode=True).decode(
+            textPart.get_content_charset()
+        )
+        htmlText = htmlPart.get_payload(decode=True).decode(
+            htmlPart.get_content_charset()
+        )
+
+        self.assertIn(u"Us\u00e9r One", plainText)
+        self.assertIn(u'<a href="mailto:user01 at localhost">Us\u00e9r One</a>',
+                      htmlText)
+
+        # The same assertion, but with the organizer's form.
+        self.assertIn(
+            u'<a href="mailto:user01 at localhost">User Z\u00e9ro One</a>',
+            htmlText)
+
+
+    def test_emailQuoting(self):
+        """
+        L{MailHandler.generateEmail} will HTML-quote all relevant fields in the
+        HTML part, but not the text/plain part.
+        """
+        _ignore_msgID, message = self.generateSampleEmail()
+        htmlPart = partByType(message, "text/html").get_payload(decode=True)
+        plainPart = partByType(message, "text/plain").get_payload(decode=True)
+        expectedPlain = 'awesome description with "<" and "&"'
+        expectedHTML = expectedPlain.replace("&", "&amp;").replace("<", "&lt;")
+
+        self.assertIn(expectedPlain, plainPart)
+        self.assertIn(expectedHTML, htmlPart)
+
+
+    def test_stringFormatTemplateLoader(self):
+        """
+        L{StringFormatTemplateLoader.load} will convert a template with
+        C{%(x)s}-format slots by converting it to a template with C{<t:slot
+        name="x" />} slots, and a renderer on the document element named
+        according to the constructor argument.
+        """
+        class StubElement(Element):
+            loader = StringFormatTemplateLoader(
+                lambda : StringIO(
+                    "<test><alpha>%(slot1)s</alpha>%(other)s</test>"
+                ),
+                "testRenderHere"
+            )
+
+            @renderer
+            def testRenderHere(self, request, tag):
+                return tag.fillSlots(slot1="hello",
+                                     other="world")
+        result = []
+        flattenString(None, StubElement()).addCallback(result.append)
+        self.assertEquals(result,
+                          ["<test><alpha>hello</alpha>world</test>"])
+
+
+    def test_templateLoaderWithAttributes(self):
+        """
+        L{StringFormatTemplateLoader.load} will convert a template with
+        C{%(x)s}-format slots inside attributes into t:attr elements containing
+        t:slot slots.
+        """
+        class StubElement(Element):
+            loader = StringFormatTemplateLoader(
+                lambda : StringIO(
+                    '<test><alpha beta="before %(slot1)s after">inner</alpha>'
+                    '%(other)s</test>'
+                ),
+                "testRenderHere"
+            )
+
+            @renderer
+            def testRenderHere(self, request, tag):
+                return tag.fillSlots(slot1="hello",
+                                     other="world")
+        result = []
+        flattenString(None, StubElement()).addCallback(result.append)
+        self.assertEquals(result,
+                          ['<test><alpha beta="before hello after">'
+                           'inner</alpha>world</test>'])
+
+
+    def test_templateLoaderTagSoup(self):
+        """
+        L{StringFormatTemplateLoader.load} will convert a template with
+        C{%(x)s}-format slots into t:slot slots, and render a well-formed output
+        document, even if the input is malformed (i.e. missing necessary closing
+        tags).
+        """
+        class StubElement(Element):
+            loader = StringFormatTemplateLoader(
+                lambda : StringIO(
+                    '<test><alpha beta="before %(slot1)s after">inner</alpha>'
+                    '%(other)s'
+                ),
+                "testRenderHere"
+            )
+
+            @renderer
+            def testRenderHere(self, request, tag):
+                return tag.fillSlots(slot1="hello",
+                                     other="world")
+        result = []
+        flattenString(None, StubElement()).addCallback(result.append)
+        self.assertEquals(result,
+                          ['<test><alpha beta="before hello after">'
+                           'inner</alpha>world</test>'])
+
+def partByType(message, contentType):
+    """
+    Retrieve a MIME part from an L{email.message.Message} based on a content
+    type.
+    """
+    for part in message.walk():
+        if part.get_content_type() == contentType:
+            return part
+    raise KeyError(contentType)

Modified: CalendarServer/trunk/twistedcaldav/scheduling/ischedule/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/scheduling/ischedule/resource.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/scheduling/ischedule/resource.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -30,6 +30,7 @@
 from twistedcaldav.scheduling.ischedule.scheduler import IScheduleScheduler
 from txdav.xml import element as davxml
 import twistedcaldav.scheduling.ischedule.xml  as ischedulexml
+from twistedcaldav.directory.util import transactionFromRequest
 from twistedcaldav.scheduling.ischedule.dkim import ISCHEDULE_CAPABILITIES
 
 __all__ = [
@@ -211,7 +212,7 @@
         scheduler = IScheduleScheduler(request, self)
 
         # Need a transaction to work with
-        txn = self._newStore.newTransaction("new transaction for Server To Server Inbox Resource")
+        txn = transactionFromRequest(request, self._newStore)
         request._newStoreTransaction = txn
 
         # Do the POST processing treating this as a non-local schedule

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -661,12 +661,6 @@
 
         "iMIP": {
             "Enabled"          : False, # Server-to-iMIP protocol
-            "MailGatewayServer" : "localhost",
-            "MailGatewayPort"   : 62310,
-            "Username"          : "com.apple.calendarserver", # For account injecting replies
-            "Password"          : "", # For account injecting replies
-            "GUID"              : "B86ED9D3-49BD-44F8-8F5E-C89D08753DAC", # GUID for special internal user
-            "Header"            : "x-calendarserver-internal", # HTTP header for internal authentication
             "Sending": {
                 "Server"        : "", # SMTP server to relay messages through
                 "Port"          : 587, # SMTP server port to relay messages through
@@ -735,16 +729,8 @@
     "Notifications" : {
         "Enabled": False,
         "CoalesceSeconds" : 3,
-        "InternalNotificationHost" : "localhost",
-        "InternalNotificationPort" : 62309,
-        "BindAddress" : "127.0.0.1",
 
         "Services" : {
-            "SimpleLineNotifier" : {
-                "Service" : "twistedcaldav.notify.SimpleLineNotifierService",
-                "Enabled" : False,
-                "Port" : 62308,
-            },
             "ApplePushNotifier" : {
                 "Service" : "calendarserver.push.applepush.ApplePushNotifierService",
                 "Enabled" : False,
@@ -778,37 +764,12 @@
             },
             "AMPNotifier" : {
                 "Service" : "calendarserver.push.amppush.AMPPushNotifierService",
-                "Enabled" : True,
+                "Enabled" : False,
                 "Port" : 62311,
                 "EnableStaggering" : False,
                 "StaggerSeconds" : 3,
+                "DataHost" : "",
             },
-            "XMPPNotifier" : {
-                "Service" : "twistedcaldav.notify.XMPPNotifierService",
-                "Enabled" : False,
-                "Host" : "", # "xmpp.host.name"
-                "Port" : 5222,
-                "JID" : "", # "jid at xmpp.host.name/resource"
-                "Password" : "",
-                "ServiceAddress" : "", # "pubsub.xmpp.host.name"
-                "CalDAV" : {
-                    "APSBundleID" : "",
-                    "SubscriptionURL" : "",
-                    "APSEnvironment" : "PRODUCTION",
-                },
-                "CardDAV" : {
-                    "APSBundleID" : "",
-                    "SubscriptionURL" : "",
-                    "APSEnvironment" : "PRODUCTION",
-                },
-                "NodeConfiguration" : {
-                    "pubsub#deliver_payloads" : "1",
-                    "pubsub#persist_items" : "1",
-                },
-                "KeepAliveSeconds" : 120,
-                "HeartbeatMinutes" : 30,
-                "AllowedJIDs": [],
-            },
         }
     },
 
@@ -1425,39 +1386,18 @@
                 except KeychainPasswordNotFound:
                     # The password doesn't exist in the keychain.
                     log.info("%s APN certificate passphrase not found in keychain" % (protocol,))
-
+                    
         if (
-            service["Service"] == "twistedcaldav.notify.XMPPNotifierService" and
+            service["Service"] == "calendarserver.push.amppush.AMPPushNotifierService" and
             service["Enabled"]
         ):
-            # If we already have the password, don't fetch it again
-            if service["Password"]:
-                continue
+            # The default for apple push DataHost is ServerHostName
+            if service["DataHost"] == "":
+                service["DataHost"] = configDict.ServerHostName
 
-            # Get password from keychain.  If not there, fall back to what
-            # is in the plist.
-            try:
-                password = getPasswordFromKeychain(service["JID"])
-                service["Password"] = password
-                log.info("XMPP password retreived from keychain")
-            except KeychainAccessError:
-                # The system doesn't support keychain
-                pass
-            except KeychainPasswordNotFound:
-                # The password doesn't exist in the keychain.
-                log.info("XMPP password not found in keychain")
 
-            # Check for empty fields
-            for key, value in service.iteritems():
-                if not value and key not in (
-                    "AllowedJIDs", "HeartbeatMinutes", "Password",
-                    "SubscriptionURL", "APSBundleID"
-                ):
-                    raise ConfigurationError("Invalid %s for XMPPNotifierService: %r"
-                                             % (key, value))
 
 
-
 def _updateScheduling(configDict, reloading=False):
     #
     # Scheduling
@@ -1471,22 +1411,6 @@
 
     if service["Enabled"]:
 
-        # If we already have the password, don't fetch it again
-        if service["Password"]:
-            return
-
-        # Get password for the user that is allowed to inject iMIP replies
-        # to the server's /inbox; if not available, fall back to plist
-        if service["Username"]:
-            try:
-                service["Password"] = getPasswordFromKeychain(service["Username"])
-            except KeychainAccessError:
-                # The system doesn't support keychain
-                pass
-            except KeychainPasswordNotFound:
-                # The password doesn't exist in the keychain.
-                log.info("iMIP injecting password not found in keychain")
-
         for direction in ("Sending", "Receiving"):
             if service[direction].Username:
                 # Get password from keychain.  If not there, fall back to
@@ -1638,12 +1562,9 @@
     re-fetched after the process has shed privileges
     """
     iMIP = configDict.Scheduling.iMIP
-    XMPP = configDict.Notifications.Services.XMPPNotifier
     preserved = {
-        "iMIPPassword" : iMIP.Password,
         "MailSendingPassword" : iMIP.Sending.Password,
         "MailReceivingPassword" : iMIP.Receiving.Password,
-        "XMPPPassword" : XMPP.Password,
     }
     return preserved
 
@@ -1655,11 +1576,8 @@
     re-fetched after the process has shed privileges
     """
     iMIP = configDict.Scheduling.iMIP
-    XMPP = configDict.Notifications.Services.XMPPNotifier
-    iMIP.Password = preserved["iMIPPassword"]
     iMIP.Sending.Password = preserved["MailSendingPassword"]
     iMIP.Receiving.Password = preserved["MailReceivingPassword"]
-    XMPP.Password = preserved["XMPPPassword"]
 
 
 config.addResetHooks(_preserveConfig, _restoreConfig)

Modified: CalendarServer/trunk/twistedcaldav/test/test_config.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_config.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/test/test_config.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -43,18 +43,6 @@
     <string>debug</string>
   </dict>
 
-  <key>Notifications</key>
-  <dict>
-    <key>Services</key>
-    <dict>
-      <key>XMPPNotifier</key>
-      <dict>
-          <key>Password</key>
-          <string>xmpp</string>
-      </dict>
-    </dict>
-  </dict>
-
   <key>Scheduling</key>
   <dict>
     <key>iMIP</key>
@@ -168,26 +156,20 @@
         self.assertEquals(config.HTTPPort, 8008)
 
     def testPreserveAcrossReload(self):
-        self.assertEquals(config.Scheduling.iMIP.Password, "")
         self.assertEquals(config.Scheduling.iMIP.Sending.Password, "")
         self.assertEquals(config.Scheduling.iMIP.Receiving.Password, "")
-        self.assertEquals(config.Notifications.Services.XMPPNotifier.Password, "")
 
         config.load(self.testConfig)
 
-        self.assertEquals(config.Scheduling.iMIP.Password, "imip")
         self.assertEquals(config.Scheduling.iMIP.Sending.Password, "sending")
         self.assertEquals(config.Scheduling.iMIP.Receiving.Password, "receiving")
-        self.assertEquals(config.Notifications.Services.XMPPNotifier.Password, "xmpp")
 
         writePlist({}, self.testConfig)
 
         config.reload()
 
-        self.assertEquals(config.Scheduling.iMIP.Password, "imip")
         self.assertEquals(config.Scheduling.iMIP.Sending.Password, "sending")
         self.assertEquals(config.Scheduling.iMIP.Receiving.Password, "receiving")
-        self.assertEquals(config.Notifications.Services.XMPPNotifier.Password, "xmpp")
 
     def testSetAttr(self):
         self.assertNotIn("BindAddresses", config.__dict__)

Deleted: CalendarServer/trunk/twistedcaldav/test/test_notify.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_notify.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/test/test_notify.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -1,627 +0,0 @@
-##
-# Copyright (c) 2008-2013 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from twisted.internet.task import Clock
-from twisted.words.protocols.jabber.client import IQ
-from twisted.words.protocols.jabber.error import StanzaError
-from twistedcaldav.notify import Notifier, NotifierFactory
-from twistedcaldav.notify import NotificationClientFactory, Coalescer
-from twistedcaldav.notify import SimpleLineNotifier, SimpleLineNotificationFactory
-from twistedcaldav.notify import SimpleLineNotificationProtocol
-from twistedcaldav.notify import XMPPNotifier, XMPPNotificationFactory
-from twistedcaldav.notify import getPubSubConfiguration, getPubSubAPSConfiguration
-from twistedcaldav.notify import getXMPPSettings
-from twistedcaldav.config import Config
-from twistedcaldav.stdconfig import DEFAULT_CONFIG, PListConfigProvider
-from twistedcaldav.test.util import TestCase
-
-
-class StubResource(object):
-
-    def __init__(self, id):
-        self._id = id
-
-    def resourceID(self):
-        return self._id
-
-
-
-class NotifierTests(TestCase):
-
-    def test_notifier(self):
-        enabledConfig = Config(PListConfigProvider(DEFAULT_CONFIG))
-        enabledConfig.Notifications["Enabled"] = True
-        notifier = Notifier(None, id="test")
-
-        self.assertEquals(notifier._ids, {"default": "test"})
-        clone = notifier.clone(label="alt", id="altID")
-        self.assertEquals("altID", clone.getID(label="alt"))
-        self.assertEquals(clone._ids, {
-            "default" : "test",
-            "alt"     : "altID",
-        })
-        self.assertEquals("test", notifier.getID())
-        self.assertEquals(notifier._ids, {
-            "default" : "test",
-        })
-        self.assertEquals(None, notifier.getID(label="notthere"))
-
-        notifier = Notifier(None, id="urn:uuid:foo")
-        self.assertEquals("foo", notifier.getID())
-
-        notifier.disableNotify()
-        self.assertEquals(notifier._notify, False)
-        notifier.enableNotify(None)
-        self.assertEquals(notifier._notify, True)
-
-        notifier = Notifier(None, id="test", prefix="CalDAV")
-        self.assertEquals("CalDAV|test", notifier.getID())
-
-
-
-class NotificationClientFactoryTests(TestCase):
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.client = StubNotificationClient(None, None)
-        self.factory = NotificationClientFactory(self.client)
-        self.factory.protocol = StubNotificationClientProtocol
-
-    def test_connect(self):
-        self.assertEquals(self.factory.isReady(), False)
-        protocol = self.factory.buildProtocol(None)
-        protocol.connectionMade()
-        self.assertEquals(self.client.observers, set([protocol]))
-        self.assertEquals(self.factory.isReady(), True)
-
-        protocol.connectionLost(None)
-        self.assertEquals(self.client.observers, set())
-        self.assertEquals(self.factory.isReady(), False)
-
-
-class StubNotificationClient(object):
-
-    def __init__(self, host, port, reactor=None):
-        self.lines = []
-        self.observers = set()
-
-    def send(self, op, id):
-        self.lines.append(id)
-
-    def addObserver(self, observer):
-        self.observers.add(observer)
-
-    def removeObserver(self, observer):
-        self.observers.remove(observer)
-
-    def connectionMade(self):
-        pass
-
-    def clear(self):
-        self.lines = []
-
-class StubNotificationClientProtocol(object):
-
-    def __init__(self):
-        self.lines = []
-
-    def sendLine(self, line):
-        self.lines.append(line)
-
-    def connectionMade(self):
-        self.client.addObserver(self)
-        self.factory.connectionMade()
-
-    def connectionLost(self, reason):
-        self.client.removeObserver(self)
-        self.factory.connected = False
-
-
-class NotifierFactoryTests(TestCase):
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.client = NotifierFactory(None, None, reactor=Clock())
-        self.client.factory = StubNotificationClientFactory()
-
-    def test_sendWhileNotConnected(self):
-        self.client.send("update", "a")
-        self.assertEquals(self.client.queued, set(["update a"]))
-
-    def test_sendWhileConnected(self):
-        protocol = StubNotificationClientProtocol()
-        self.client.addObserver(protocol)
-        self.client.factory.connected = True
-        self.client.send("update", "a")
-        self.assertEquals(self.client.queued, set())
-        self.assertEquals(protocol.lines, ["update a"])
-
-    def test_sendQueue(self):
-        self.client.send("update", "a")
-        self.assertEquals(self.client.queued, set(["update a"]))
-        protocol = StubNotificationClientProtocol()
-        self.client.addObserver(protocol)
-        self.client.factory.connected = True
-        self.client.connectionMade()
-        self.assertEquals(protocol.lines, ["update a"])
-        self.assertEquals(self.client.queued, set())
-
-
-class StubNotificationClientFactory(object):
-
-    def __init__(self):
-        self.connected = False
-
-    def isReady(self):
-        return self.connected
-
-
-class CoalescerTests(TestCase):
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.clock = Clock()
-        self.notifier = StubNotifier()
-        self.coalescer = Coalescer([self.notifier], reactor=self.clock)
-
-    def test_delayedNotifications(self):
-        self.coalescer.add("update", "A")
-        self.assertEquals(self.notifier.notifications, [])
-        self.clock.advance(5)
-        self.assertEquals(self.notifier.notifications, ["A"])
-
-    def test_removeDuplicates(self):
-        self.coalescer.add("update", "A")
-        self.coalescer.add("update", "A")
-        self.clock.advance(5)
-        self.assertEquals(self.notifier.notifications, ["A"])
-
-
-class StubNotifier(object):
-
-    def __init__(self):
-        self.notifications = []
-        self.observers = set()
-        self.playbackHistory = []
-
-    def enqueue(self, op, id):
-        self.notifications.append(id)
-
-    def playback(self, protocol, old_seq):
-        self.playbackHistory.append((protocol, old_seq))
-
-    def addObserver(self, observer):
-        self.observers.add(observer)
-
-    def removeObserver(self, observer):
-        self.observers.remove(observer)
-
-
-class SimpleLineNotifierTests(TestCase):
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.clock = Clock()
-        self.notifier = SimpleLineNotifier(None)
-        self.coalescer = Coalescer([self.notifier], reactor=self.clock)
-
-    def test_initialConnection(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.connectionMade(protocol)
-        self.assertEquals(protocol.lines, ["0"])
-
-    def test_subsequentConnection(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.connectionMade(protocol)
-        protocol.lines = []
-        self.notifier.connectionMade(protocol)
-        self.assertEquals(protocol.lines, [])
-
-    def test_send(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.enqueue("update", "A")
-        self.assertEquals(protocol.lines, ["1 A"])
-
-    def test_incrementSequence(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.enqueue("update", "A")
-        self.notifier.enqueue("update", "B")
-        self.assertEquals(protocol.lines, ["1 A", "2 B"])
-
-    def test_addObserver(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.enqueue("update", "A")
-        self.assertEquals(protocol.lines, ["1 A"])
-
-    def test_removeObserver(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.removeObserver(protocol)
-        self.notifier.enqueue("update", "A")
-        self.assertEquals(protocol.lines, [])
-
-    def test_multipleObservers(self):
-        protocol1 = StubProtocol()
-        protocol2 = StubProtocol()
-        self.notifier.addObserver(protocol1)
-        self.notifier.addObserver(protocol2)
-        self.notifier.enqueue("update", "A")
-        self.assertEquals(protocol1.lines, ["1 A"])
-        self.assertEquals(protocol2.lines, ["1 A"])
-
-    def test_duplicateObservers(self):
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.addObserver(protocol)
-        self.notifier.enqueue("update", "A")
-        self.assertEquals(protocol.lines, ["1 A"])
-
-    def test_playback(self):
-        self.notifier.enqueue("update", "A")
-        self.notifier.enqueue("update", "B")
-        self.notifier.enqueue("update", "C")
-        protocol = StubProtocol()
-        self.notifier.addObserver(protocol)
-        self.notifier.playback(protocol, 1)
-        self.assertEquals(protocol.lines, ["2 B", "3 C"])
-
-    def test_reset(self):
-        self.notifier.enqueue("update", "A")
-        self.assertEquals(self.notifier.history, {"A" : 1})
-        self.assertEquals(self.notifier.latestSeq, 1)
-        self.notifier.reset()
-        self.assertEquals(self.notifier.history, {})
-        self.assertEquals(self.notifier.latestSeq, 0)
-        
-
-class SimpleLineNotificationFactoryTests(TestCase):
-
-    def test_buildProtocol(self):
-        notifier = StubNotifier()
-        factory = SimpleLineNotificationFactory(notifier)
-        protocol = factory.buildProtocol(None)
-        self.assertEquals(protocol.notifier, notifier)
-        self.assertIn(protocol, notifier.observers)
-
-
-class SimpleLineNotificationProtocolTests(TestCase):
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.notifier = StubNotifier()
-        self.protocol = SimpleLineNotificationProtocol()
-        self.protocol.notifier = self.notifier
-        self.protocol.transport = StubTransport()
-        self.notifier.addObserver(self.protocol)
-
-    def test_connectionLost(self):
-        self.protocol.connectionLost(None)
-        self.assertNotIn(self.protocol, self.notifier.observers)
-
-    def test_lineReceived(self):
-        self.protocol.lineReceived("2")
-        self.assertEquals(self.notifier.playbackHistory, [(self.protocol, 2)])
-
-    def test_lineReceivedInvalid(self):
-        self.protocol.lineReceived("bogus")
-        self.assertEquals(self.notifier.playbackHistory, [])
-
-
-
-class StubProtocol(object):
-
-    def __init__(self):
-        self.lines = []
-
-    def sendLine(self, line):
-        self.lines.append(line)
-
-
-class StubTransport(object):
-
-    def getPeer(self):
-        return "peer"
-
-
-
-
-
-
-class StubXmlStream(object):
-
-    def __init__(self):
-        self.elements = []
-
-    def send(self, element):
-        self.elements.append(element)
-
-    def addOnetimeObserver(self, *args, **kwds):
-        pass
-
-    def addObserver(self, *args, **kwds):
-        pass
-
-
-class StubFailure(object):
-
-    def __init__(self, value):
-        self.value = value
-
-class XMPPNotifierTests(TestCase):
-
-    xmppEnabledConfig = Config(PListConfigProvider(DEFAULT_CONFIG))
-    xmppEnabledConfig.Notifications["Enabled"] = True
-    xmppEnabledConfig.Notifications["Services"]["XMPPNotifier"]["Enabled"] = True
-    xmppEnabledConfig.ServerHostName = "server.example.com"
-    xmppEnabledConfig.HTTPPort = 80
-
-    xmppDisabledConfig = Config(PListConfigProvider(DEFAULT_CONFIG))
-    xmppDisabledConfig.Notifications["Services"]["XMPPNotifier"]["Enabled"] = False
-
-    def setUp(self):
-        TestCase.setUp(self)
-        self.xmlStream = StubXmlStream()
-        self.settings = { "ServiceAddress" : "pubsub.example.com",
-            "NodeConfiguration" : { "pubsub#deliver_payloads" : "1" },
-            "HeartbeatMinutes" : 30,
-        }
-        self.notifier = XMPPNotifier(self.settings, reactor=Clock(),
-            configOverride=self.xmppEnabledConfig, heartbeat=False)
-        self.notifier.streamOpened(self.xmlStream)
-
-    def test_sendWhileConnected(self):
-        self.notifier.enqueue("update", "test")
-
-        iq = self.xmlStream.elements[1]
-        self.assertEquals(iq.name, "iq")
-
-        pubsubElement = list(iq.elements())[0]
-        self.assertEquals(pubsubElement.name, "pubsub")
-        self.assertEquals(pubsubElement.uri, "http://jabber.org/protocol/pubsub")
-
-        publishElement = list(pubsubElement.elements())[0]
-        self.assertEquals(publishElement.name, "publish")
-        self.assertEquals(publishElement.uri, "http://jabber.org/protocol/pubsub")
-        self.assertEquals(publishElement["node"],
-            "/server.example.com/test/")
-
-    def test_sendWhileNotConnected(self):
-        notifier = XMPPNotifier(self.settings, reactor=Clock(),
-            configOverride=self.xmppDisabledConfig)
-        notifier.enqueue("update", "/principals/__uids__/test")
-        self.assertEquals(len(self.xmlStream.elements), 1)
-
-    def test_publishNewNode(self):
-        self.notifier.publishNode("testNodeName")
-        iq = self.xmlStream.elements[1]
-        self.assertEquals(iq.name, "iq")
-
-    def test_publishReponse400(self):
-        failure = StubFailure(StanzaError("bad-request"))
-        self.assertEquals(len(self.xmlStream.elements), 1)
-        self.notifier.publishNodeFailure(failure, "testNodeName")
-        self.assertEquals(len(self.xmlStream.elements), 2)
-        iq = self.xmlStream.elements[1]
-        self.assertEquals(iq.name, "iq")
-        self.assertEquals(iq["type"], "get")
-
-        pubsubElement = list(iq.elements())[0]
-        self.assertEquals(pubsubElement.name, "pubsub")
-        self.assertEquals(pubsubElement.uri,
-            "http://jabber.org/protocol/pubsub#owner")
-        configElement = list(pubsubElement.elements())[0]
-        self.assertEquals(configElement.name, "configure")
-        self.assertEquals(configElement["node"], "testNodeName")
-
-
-    def test_publishReponse404(self):
-        self.assertEquals(len(self.xmlStream.elements), 1)
-        failure = StubFailure(StanzaError("item-not-found"))
-        self.notifier.publishNodeFailure(failure, "testNodeName")
-        self.assertEquals(len(self.xmlStream.elements), 2)
-        iq = self.xmlStream.elements[1]
-        self.assertEquals(iq.name, "iq")
-        self.assertEquals(iq["type"], "set")
-
-        pubsubElement = list(iq.elements())[0]
-        self.assertEquals(pubsubElement.name, "pubsub")
-        self.assertEquals(pubsubElement.uri,
-            "http://jabber.org/protocol/pubsub")
-        createElement = list(pubsubElement.elements())[0]
-        self.assertEquals(createElement.name, "create")
-        self.assertEquals(createElement["node"], "testNodeName")
-
-
-    def test_configureResponse(self):
-
-        def _getChild(element, name):
-            for child in element.elements():
-                if child.name == name:
-                    return child
-            return None
-
-        response = IQ(self.xmlStream, type="result")
-        pubsubElement = response.addElement("pubsub")
-        configElement = pubsubElement.addElement("configure")
-        formElement = configElement.addElement("x")
-        formElement["type"] = "form"
-        fields = [
-            ( "unknown", "don't edit me", "text-single" ),
-            ( "pubsub#deliver_payloads", "0", "boolean" ),
-            ( "pubsub#persist_items", "0", "boolean" ),
-        ]
-        expectedFields = {
-            "unknown" : "don't edit me",
-            "pubsub#deliver_payloads" : "1",
-            "pubsub#persist_items" : "1",
-        }
-        for field in fields:
-            fieldElement = formElement.addElement("field")
-            fieldElement["var"] = field[0]
-            fieldElement["type"] = field[2]
-            fieldElement.addElement("value", content=field[1])
-
-        self.assertEquals(len(self.xmlStream.elements), 1)
-        self.notifier.requestConfigurationFormSuccess(response, "testNodeName",
-            False)
-        self.assertEquals(len(self.xmlStream.elements), 2)
-
-        iq = self.xmlStream.elements[1]
-        self.assertEquals(iq.name, "iq")
-        self.assertEquals(iq["type"], "set")
-
-        pubsubElement = list(iq.elements())[0]
-        self.assertEquals(pubsubElement.name, "pubsub")
-        configElement = list(pubsubElement.elements())[0]
-        self.assertEquals(configElement.name, "configure")
-        self.assertEquals(configElement["node"], "testNodeName")
-        formElement = list(configElement.elements())[0]
-        self.assertEquals(formElement["type"], "submit")
-        for field in formElement.elements():
-            valueElement = _getChild(field, "value")
-            if valueElement is not None:
-                self.assertEquals(expectedFields[field["var"]],
-                    str(valueElement))
-
-
-    def test_sendHeartbeat(self):
-
-        xmppConfig = Config(PListConfigProvider(DEFAULT_CONFIG))
-        xmppConfig.Notifications["Enabled"] = True
-        xmppConfig.Notifications["Services"]["XMPPNotifier"]["Enabled"] = True
-        xmppConfig.ServerHostName = "server.example.com"
-        xmppConfig.HTTPPort = 80
-
-        clock = Clock()
-        xmlStream = StubXmlStream()
-        settings = { "ServiceAddress" : "pubsub.example.com", "JID" : "jid",
-            "Password" : "password", "KeepAliveSeconds" : 5,
-            "NodeConfiguration" : { "pubsub#deliver_payloads" : "1" },
-            "HeartbeatMinutes" : 30 }
-        notifier = XMPPNotifier(settings, reactor=clock, heartbeat=True,
-            roster=False, configOverride=xmppConfig)
-        factory = XMPPNotificationFactory(notifier, settings, reactor=clock,
-            keepAlive=False)
-        factory.connected(xmlStream)
-        factory.authenticated(xmlStream)
-
-        self.assertEquals(len(xmlStream.elements), 1)
-        heartbeat = xmlStream.elements[0]
-        self.assertEquals(heartbeat.name, "iq")
-
-        clock.advance(1800)
-
-        self.assertEquals(len(xmlStream.elements), 2)
-        heartbeat = xmlStream.elements[1]
-        self.assertEquals(heartbeat.name, "iq")
-
-        factory.disconnected(xmlStream)
-        clock.advance(1800)
-        self.assertEquals(len(xmlStream.elements), 2)
-
-
-
-
-class XMPPNotificationFactoryTests(TestCase):
-
-    def test_sendPresence(self):
-        clock = Clock()
-        xmlStream = StubXmlStream()
-        settings = { "ServiceAddress" : "pubsub.example.com", "JID" : "jid",
-            "NodeConfiguration" : { "pubsub#deliver_payloads" : "1" },
-            "Password" : "password", "KeepAliveSeconds" : 5 }
-        notifier = XMPPNotifier(settings, reactor=clock, heartbeat=False)
-        factory = XMPPNotificationFactory(notifier, settings, reactor=clock)
-        factory.connected(xmlStream)
-        factory.authenticated(xmlStream)
-
-        self.assertEquals(len(xmlStream.elements), 2)
-        presence = xmlStream.elements[0]
-        self.assertEquals(presence.name, "presence")
-        iq = xmlStream.elements[1]
-        self.assertEquals(iq.name, "iq")
-
-        clock.advance(5)
-
-        self.assertEquals(len(xmlStream.elements), 3)
-        presence = xmlStream.elements[2]
-        self.assertEquals(presence.name, "presence")
-
-        factory.disconnected(xmlStream)
-        clock.advance(5)
-        self.assertEquals(len(xmlStream.elements), 3)
-
-
-
-class ConfigurationTests(TestCase):
-
-    def test_disabled(self):
-        disabledConfig = Config(PListConfigProvider(DEFAULT_CONFIG))
-
-        # Overall notifications are disabled
-        disabledConfig.Notifications["Enabled"] = False
-        conf = getPubSubConfiguration(disabledConfig)
-        self.assertEquals(conf, { "enabled" : False, "host" : "" })
-        conf = getXMPPSettings(disabledConfig)
-        self.assertEquals(conf, None)
-
-        # Overall notifications are enabled, but XMPP disabled
-        disabledConfig.Notifications["Enabled"] = True
-        settings = getXMPPSettings(disabledConfig)
-        self.assertEquals(settings, None)
-
-        # Overall notifications are enabled, XMPP enabled, but no APS
-        service = disabledConfig.Notifications["Services"]["XMPPNotifier"]
-        service.Enabled = True
-        conf = getPubSubAPSConfiguration("CalDAV|foo", disabledConfig)
-        self.assertEquals(conf, None)
-
-    def test_enabled(self):
-        enabledConfig = Config(PListConfigProvider(DEFAULT_CONFIG))
-        enabledConfig.Notifications["Enabled"] = True
-        service = enabledConfig.Notifications["Services"]["XMPPNotifier"]
-        service.Enabled = True
-        service.Host = "example.com"
-        service.Port = 5222
-        service.ServiceAddress = "pubsub.example.com"
-        service.CalDAV.APSBundleID = "CalDAVAPSBundleID"
-        service.CalDAV.SubscriptionURL = "CalDAVSubscriptionURL"
-        conf = getPubSubConfiguration(enabledConfig)
-        self.assertEquals(conf, {'heartrate': 30, 'service': 'pubsub.example.com', 'xmpp-server': 'example.com', 'enabled': True, 'host': '', 'port': 0} )
-        conf = getPubSubAPSConfiguration("CalDAV|foo", enabledConfig)
-        self.assertEquals(conf, {'SubscriptionURL': 'CalDAVSubscriptionURL', 'APSBundleID': 'CalDAVAPSBundleID', 'APSEnvironment' : 'PRODUCTION'} )
-        conf = getPubSubAPSConfiguration("noprefix", enabledConfig)
-        self.assertEquals(conf, None)
-        conf = getPubSubAPSConfiguration("UnknownPrefix|foo", enabledConfig)
-        self.assertEquals(conf, None)
-
-    def test_allowedInRoster(self):
-        """
-        Our own JID is implicitly included in AllowedJIDs
-        """
-        settings = {
-            "JID" : "test1 at example.com",
-            "AllowedJIDs" : ["test2 at example.com"]
-        }
-        notifier = XMPPNotifier(settings, heartbeat=False)
-        self.assertTrue(notifier.allowedInRoster("test1 at example.com"))
-        self.assertTrue(notifier.allowedInRoster("test2 at example.com"))
-        self.assertFalse(notifier.allowedInRoster("test3 at example.com"))

Modified: CalendarServer/trunk/twistedcaldav/test/test_resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_resource.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/test/test_resource.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -73,66 +73,20 @@
         resource = CalendarHomeResource(None, None, None, StubHome())
         self.assertTrue(('http://calendarserver.org/ns/', 'push-transports') in resource.liveProperties())
         self.assertTrue(('http://calendarserver.org/ns/', 'pushkey') in resource.liveProperties())
-        self.assertTrue(('http://calendarserver.org/ns/', 'xmpp-uri') in resource.liveProperties())
-        self.assertTrue(('http://calendarserver.org/ns/', 'xmpp-heartbeat-uri') in resource.liveProperties())
-        self.assertTrue(('http://calendarserver.org/ns/', 'xmpp-server') in resource.liveProperties())
 
 
     def test_addressBookHomeliveProperties(self):
         resource = AddressBookHomeResource(None, None, None, StubHome())
         self.assertTrue(('http://calendarserver.org/ns/', 'push-transports') in resource.liveProperties())
         self.assertTrue(('http://calendarserver.org/ns/', 'pushkey') in resource.liveProperties())
-        self.assertTrue(('http://calendarserver.org/ns/', 'xmpp-uri') not in resource.liveProperties())
-        self.assertTrue(('http://calendarserver.org/ns/', 'xmpp-heartbeat-uri') not in resource.liveProperties())
-        self.assertTrue(('http://calendarserver.org/ns/', 'xmpp-server') not in resource.liveProperties())
 
     def test_notificationCollectionLiveProperties(self):
         resource = NotificationCollectionResource()
         self.assertTrue(('http://calendarserver.org/ns/', 'getctag') in resource.liveProperties())
 
 
-    @inlineCallbacks
-    def test_push404(self):
-        """
-        If push is configured, yet we can't communicate with the XMPP server
-        for whatever reason, readProperty on the various push-related properties
-        should return None
-        """
-        resource = CalendarHomeResource(None, None, None, StubHome())
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'push-transports'), None)), None)
 
-        self.patch(config, "ServerHostName", "cal.example.com")
-        self.patch(config, "SSLPort", 8443)
-        self.patch(config.Notifications, "Enabled", True)
-        self.patch(config.Notifications.Services, "XMPPNotifier", 
-            {
-                "Enabled" : True,
-                "Host" : "xmpp.example.com",
-                "Port" : 5218,
-                "ServiceAddress" : "pubsub.xmpp.example.com",
-                "Service" : "twistedcaldav.notify.XMPPNotifierService",
-                "HeartbeatMinutes" : 30,
-            }
-        )
 
-        # Verify that when push is "working" we get a value
-        resource._newStoreHome.setPushWorking(True)
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'push-transports'), None)).toxml(), "<?xml version='1.0' encoding='UTF-8'?>\n<push-transports xmlns='http://calendarserver.org/ns/'>\r\n  <transport type='XMPP'>\r\n    <xmpp-server>xmpp.example.com:5218</xmpp-server>\r\n    <xmpp-uri>xmpp:pubsub.xmpp.example.com?pubsub;node=/cal.example.com/xyzzy/</xmpp-uri>\r\n  </transport>\r\n</push-transports>")
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'pushkey'), None)).toxml(), "<?xml version='1.0' encoding='UTF-8'?>\n<pushkey xmlns='http://calendarserver.org/ns/'>xyzzy</pushkey>")
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'xmpp-uri'), None)).toxml(), "<?xml version='1.0' encoding='UTF-8'?>\n<xmpp-uri xmlns='http://calendarserver.org/ns/'>xmpp:pubsub.xmpp.example.com?pubsub;node=/cal.example.com/xyzzy/</xmpp-uri>")
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'xmpp-heartbeat-uri'), None)).toxml(), "<?xml version='1.0' encoding='UTF-8'?>\n<xmpp-heartbeat xmlns='http://calendarserver.org/ns/'>\r\n  <xmpp-heartbeat-uri>xmpp:pubsub.xmpp.example.com?pubsub;node=/cal.example.com/</xmpp-heartbeat-uri>\r\n  <xmpp-heartbeat-minutes>30</xmpp-heartbeat-minutes>\r\n</xmpp-heartbeat>")
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'xmpp-server'), None)).toxml(), "<?xml version='1.0' encoding='UTF-8'?>\n<xmpp-server xmlns='http://calendarserver.org/ns/'>xmpp.example.com:5218</xmpp-server>")
-
-        # Verify that when push is "not working" we get None
-        resource._newStoreHome.setPushWorking(False)
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'push-transports'), None)), None)
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'pushkey'), None)), None)
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'xmpp-uri'), None)), None)
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'xmpp-heartbeat-uri'), None)), None)
-        self.assertEqual((yield resource.readProperty(('http://calendarserver.org/ns/', 'xmpp-server'), None)), None)
-
-
-
 class OwnershipTests(TestCase):
     """
     L{CalDAVResource.isOwner} determines if the authenticated principal of the

Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -66,7 +66,9 @@
 from calendarserver.tools.util import getDirectory
 
 from twext.python.parallel import Parallelizer
+from twistedcaldav.scheduling.imip.mailgateway import migrateTokensToStore
 
+
 deadPropertyXattrPrefix = namedAny(
     "txdav.base.propertystore.xattr.PropertyStore.deadPropertyXattrPrefix"
 )
@@ -1048,6 +1050,8 @@
         Start the service.
         """
 
+        directory = directoryFromConfig(self.config)
+
         # Load proxy assignments from XML if specified
         if self.config.ProxyLoadFromFile:
             proxydbClass = namedClass(self.config.ProxyDBService.type)
@@ -1063,7 +1067,6 @@
             if proxydb is None:
                 proxydbClass = namedClass(self.config.ProxyDBService.type)
                 proxydb = proxydbClass(**self.config.ProxyDBService.params)
-            directory = directoryFromConfig(self.config)
 
             updater = GroupMembershipCacheUpdater(proxydb,
                 directory, self.config.GroupCaching.ExpireSeconds,
@@ -1082,7 +1085,11 @@
         yield self.processInboxItems()
         self.store.setMigrating(False)
 
+        # Migrate mail tokens from sqlite to store
+        yield migrateTokensToStore(self.config.DataRoot, self.store)
 
+
+
     @inlineCallbacks
     def processInboxItems(self):
         """

Modified: CalendarServer/trunk/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/common.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/common.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -445,8 +445,8 @@
         # Make sure notification fired after commit
         self.assertEquals(self.notifierFactory.history,
             [
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/notification"),
+                "CalDAV|home1",
+                "CalDAV|home1/notification",
             ]
         )
 
@@ -463,8 +463,8 @@
         # Make sure notification fired after commit
         self.assertEquals(self.notifierFactory.history,
             [
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/notification"),
+                "CalDAV|home1",
+                "CalDAV|home1/notification",
             ]
         )
 
@@ -536,16 +536,7 @@
         self.assertEquals(name, "/CalDAV/example.com/home1/")
 
 
-    @inlineCallbacks
-    def test_nodeNameFailure(self):
-        # The StubNodeCacher is set up to fail when the node name has the
-        # word "fail" in it, for testing the failure mode:
-        home = yield self.transactionUnderTest().calendarHomeWithUID("fail",
-            create=True)
-        name = yield home.nodeName()
-        self.assertEquals(name, None)
 
-
     @inlineCallbacks
     def test_displayNameNone(self):
         """
@@ -716,7 +707,7 @@
         yield self.commit()
 
         # Make sure notification fired after commit
-        self.assertTrue(("update", "CalDAV|home1") in self.notifierFactory.history)
+        self.assertTrue("CalDAV|home1" in self.notifierFactory.history)
 
         # Make sure it's available in a new transaction; i.e. test the commit.
         home = yield self.homeUnderTest()
@@ -765,12 +756,12 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/calendar_1"),
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/calendar_2"),
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/calendar_empty"),
+                "CalDAV|home1",
+                "CalDAV|home1/calendar_1",
+                "CalDAV|home1",
+                "CalDAV|home1/calendar_2",
+                "CalDAV|home1",
+                "CalDAV|home1/calendar_empty",
             ]
         )
 
@@ -943,8 +934,8 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/calendar_1"),
+                "CalDAV|home1",
+                "CalDAV|home1/calendar_1",
             ]
         )
 
@@ -1495,8 +1486,8 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/calendar_1"),
+                "CalDAV|home1",
+                "CalDAV|home1/calendar_1",
             ]
         )
 
@@ -1612,8 +1603,8 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CalDAV|home1"),
-                ("update", "CalDAV|home1/calendar_1"),
+                "CalDAV|home1",
+                "CalDAV|home1/calendar_1",
             ]
         )
 

Modified: CalendarServer/trunk/txdav/carddav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/common.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/common.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -330,7 +330,7 @@
         yield self.commit()
 
         # Make sure notification fired after commit
-        self.assertTrue(("update", "CardDAV|home1") in self.notifierFactory.history)
+        self.assertTrue("CardDAV|home1" in self.notifierFactory.history)
 
         # Make sure it's available in a new transaction; i.e. test the commit.
         home = yield self.homeUnderTest()
@@ -379,12 +379,12 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CardDAV|home1"),
-                ("update", "CardDAV|home1/addressbook_1"),
-                ("update", "CardDAV|home1"),
-                ("update", "CardDAV|home1/addressbook_2"),
-                ("update", "CardDAV|home1"),
-                ("update", "CardDAV|home1/addressbook_empty")
+                "CardDAV|home1",
+                "CardDAV|home1/addressbook_1",
+                "CardDAV|home1",
+                "CardDAV|home1/addressbook_2",
+                "CardDAV|home1",
+                "CardDAV|home1/addressbook_empty"
             ]
         )
 
@@ -515,8 +515,8 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CardDAV|home1"),
-                ("update", "CardDAV|home1/addressbook_1"),
+                "CardDAV|home1",
+                "CardDAV|home1/addressbook_1",
             ]
         )
 
@@ -705,8 +705,8 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CardDAV|home1"),
-                ("update", "CardDAV|home1/addressbook_1"),
+                "CardDAV|home1",
+                "CardDAV|home1/addressbook_1",
             ]
         )
 
@@ -821,8 +821,8 @@
         self.assertEquals(
             self.notifierFactory.history,
             [
-                ("update", "CardDAV|home1"),
-                ("update", "CardDAV|home1/addressbook_1"),
+                "CardDAV|home1",
+                "CardDAV|home1/addressbook_1",
             ]
         )
 

Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/common/datastore/file.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -117,15 +117,25 @@
         self.quota = quota
         self._migrating = False
         self._enableNotifications = True
+        self._newTransactionCallbacks = set()
 
+    def callWithNewTransactions(self, callback):
+        """
+        Registers a method to be called whenever a new transaction is
+        created.
 
+        @param callback: callable taking a single argument, a transaction
+        """
+        self._newTransactionCallbacks.add(callback)
+
+
     def newTransaction(self, name='no name'):
         """
         Create a new transaction.
 
         @see: L{Transaction}
         """
-        return self._transactionClass(
+        txn = self._transactionClass(
             self,
             name,
             self.enableCalendars,
@@ -133,6 +143,9 @@
             self._notifierFactory if self._enableNotifications else None,
             self._migrating,
         )
+        for callback in self._newTransactionCallbacks:
+            callback(txn)
+        return txn
 
 
     @inlineCallbacks
@@ -337,6 +350,26 @@
         return NotImplementedError
 
 
+    def imipCreateToken(self, organizer, attendee, icaluid, token=None):
+        return NotImplementedError
+
+
+    def imipLookupByToken(self, token):
+        return NotImplementedError
+
+
+    def imipGetToken(self, organizer, attendee, icaluid):
+        return NotImplementedError
+
+
+    def imipRemoveToken(self, token):
+        return NotImplementedError
+
+
+    def purgeOldIMIPTokens(self, olderThan):
+        return NotImplementedError
+
+
     def isNotifiedAlready(self, obj):
         return obj in self._notifiedAlready
 

Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -64,7 +64,8 @@
 from txdav.common.icommondatastore import HomeChildNameNotAllowedError, \
     HomeChildNameAlreadyExistsError, NoSuchHomeChildError, \
     ObjectResourceNameNotAllowedError, ObjectResourceNameAlreadyExistsError, \
-    NoSuchObjectResourceError, AllRetriesFailed, InvalidSubscriptionValues
+    NoSuchObjectResourceError, AllRetriesFailed, InvalidSubscriptionValues, \
+    InvalidIMIPTokenValues
 from txdav.common.inotifications import INotificationCollection, \
     INotificationObject
 
@@ -176,6 +177,7 @@
         self.queuer = LocalQueuer(self.newTransaction)
         self._migrating = False
         self._enableNotifications = True
+        self._newTransactionCallbacks = set()
 
         if cacheQueries:
             self.queryCacher = QueryCacher(cachePool=cachePool,
@@ -189,6 +191,16 @@
         __import__("txdav.carddav.datastore.sql")
 
 
+    def callWithNewTransactions(self, callback):
+        """
+        Registers a method to be called whenever a new transaction is
+        created.
+
+        @param callback: callable taking a single argument, a transaction
+        """
+        self._newTransactionCallbacks.add(callback)
+
+
     @inlineCallbacks
     def _withEachHomeDo(self, homeTable, homeFromTxn, action, batchSize):
         """
@@ -248,6 +260,8 @@
         if self.logTransactionWaits or self.timeoutTransactions:
             CommonStoreTransactionMonitor(txn, self.logTransactionWaits,
                                           self.timeoutTransactions)
+        for callback in self._newTransactionCallbacks:
+            callback(txn)
         return txn
 
 
@@ -716,6 +730,105 @@
         return self._apnSubscriptionsBySubscriberQuery.on(self, subscriberGUID=guid)
 
 
+    # Create IMIP token
+
+    @classproperty
+    def _insertIMIPTokenQuery(cls): #@NoSelf
+        imip = schema.IMIP_TOKENS
+        return Insert({imip.TOKEN: Parameter("token"),
+                       imip.ORGANIZER: Parameter("organizer"),
+                       imip.ATTENDEE: Parameter("attendee"),
+                       imip.ICALUID: Parameter("icaluid"),
+                      })
+
+    @inlineCallbacks
+    def imipCreateToken(self, organizer, attendee, icaluid, token=None):
+        if not (organizer and attendee and icaluid):
+            raise InvalidIMIPTokenValues()
+
+        if token is None:
+            token = str(uuid4())
+
+        try:
+            yield self._insertIMIPTokenQuery.on(self,
+                token=token, organizer=organizer, attendee=attendee,
+                icaluid=icaluid)
+        except Exception:
+            # TODO: is it okay if someone else created the same row just now?
+            pass
+        returnValue(token)
+
+    # Lookup IMIP organizer+attendee+icaluid for token
+
+    @classproperty
+    def _selectIMIPTokenByTokenQuery(cls): #@NoSelf
+        imip = schema.IMIP_TOKENS
+        return Select([imip.ORGANIZER, imip.ATTENDEE, imip.ICALUID], From=imip,
+                      Where=(imip.TOKEN == Parameter("token")))
+
+    def imipLookupByToken(self, token):
+        return self._selectIMIPTokenByTokenQuery.on(self, token=token)
+
+    # Lookup IMIP token for organizer+attendee+icaluid
+
+    @classproperty
+    def _selectIMIPTokenQuery(cls): #@NoSelf
+        imip = schema.IMIP_TOKENS
+        return Select([imip.TOKEN], From=imip,
+                      Where=(imip.ORGANIZER == Parameter("organizer")).And(
+                             imip.ATTENDEE == Parameter("attendee")).And(
+                             imip.ICALUID == Parameter("icaluid")))
+    @classproperty
+    def _updateIMIPTokenQuery(cls): #@NoSelf
+        imip = schema.IMIP_TOKENS
+        return Update({imip.ACCESSED: utcNowSQL,},
+                      Where=(imip.ORGANIZER == Parameter("organizer")).And(
+                             imip.ATTENDEE == Parameter("attendee")).And(
+                             imip.ICALUID == Parameter("icaluid")))
+
+
+    @inlineCallbacks
+    def imipGetToken(self, organizer, attendee, icaluid):
+        row = (yield self._selectIMIPTokenQuery.on(self, organizer=organizer,
+            attendee=attendee, icaluid=icaluid))
+        if row:
+            token = row[0][0]
+            # update the timestamp
+            yield self._updateIMIPTokenQuery.on(self, organizer=organizer,
+                attendee=attendee, icaluid=icaluid)
+        else:
+            token = None
+        returnValue(token)
+
+    # Remove IMIP token
+
+    @classproperty
+    def _removeIMIPTokenQuery(cls): #@NoSelf
+        imip = schema.IMIP_TOKENS
+        return Delete(From=imip,
+                      Where=(imip.TOKEN == Parameter("token")))
+
+    def imipRemoveToken(self, token):
+        return self._removeIMIPTokenQuery.on(self, token=token)
+
+    # Purge old IMIP tokens
+
+    @classproperty
+    def _purgeOldIMIPTokensQuery(cls): #@NoSelf
+        imip = schema.IMIP_TOKENS
+        return Delete(From=imip,
+                      Where=(imip.ACCESSED < Parameter("olderThan")))
+
+    def purgeOldIMIPTokens(self, olderThan):
+        """
+        @type olderThan: datetime
+        """
+        return self._purgeOldIMIPTokensQuery.on(self,
+            olderThan=olderThan)
+
+    # End of IMIP
+
+
     def preCommit(self, operation):
         """
         Run things before C{commit}.  (Note: only provided by SQL

Added: CalendarServer/trunk/txdav/common/datastore/sql_schema/ReadMe.txt
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/ReadMe.txt	                        (rev 0)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/ReadMe.txt	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,9 @@
+Steps for Modifying the Schema
+==============================
+
+1. Before changing current.sql, take note of the VERSION number in the schema
+2. Copy current.sql into old/postgres-dialect/vNNN.sql (where NNN is the VERSION number prior to your changes)
+3. Copy current-oracle-dialect.sql into old/oracle-dialect/vNNN.sql (where NNN is the VERSION number prior to your changes)
+4. Make your changes to current.sql, bumping up the VERSION number
+5. Use the sql_tables.py to generate the oracle version and save as current-oracle-dialect.sql
+6. Write upgrade scripts within upgrades/postgres-dialect and upgrades/oracle-dialect

Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql	2013-02-16 00:11:19 UTC (rev 10748)
@@ -2,6 +2,7 @@
 create sequence INSTANCE_ID_SEQ;
 create sequence ATTACHMENT_ID_SEQ;
 create sequence REVISION_SEQ;
+create sequence WORKITEM_SEQ;
 create table NODE_INFO (
     "HOSTNAME" nvarchar2(255),
     "PID" integer not null,
@@ -58,7 +59,7 @@
 create table CALENDAR_BIND (
     "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
     "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
-    "CALENDAR_RESOURCE_NAME" nvarchar2(255) not null,
+    "CALENDAR_RESOURCE_NAME" nvarchar2(255),
     "BIND_MODE" integer not null,
     "BIND_STATUS" integer not null,
     "MESSAGE" nclob, 
@@ -207,7 +208,7 @@
 create table ADDRESSBOOK_BIND (
     "ADDRESSBOOK_HOME_RESOURCE_ID" integer not null references ADDRESSBOOK_HOME,
     "ADDRESSBOOK_RESOURCE_ID" integer not null references ADDRESSBOOK on delete cascade,
-    "ADDRESSBOOK_RESOURCE_NAME" nvarchar2(255) not null,
+    "ADDRESSBOOK_RESOURCE_NAME" nvarchar2(255),
     "BIND_MODE" integer not null,
     "BIND_STATUS" integer not null,
     "MESSAGE" nclob, 
@@ -264,15 +265,50 @@
     primary key("TOKEN", "RESOURCE_KEY")
 );
 
+create table IMIP_TOKENS (
+    "TOKEN" nvarchar2(255),
+    "ORGANIZER" nvarchar2(255),
+    "ATTENDEE" nvarchar2(255),
+    "ICALUID" nvarchar2(255),
+    "ACCESSED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC', 
+    primary key("ORGANIZER", "ATTENDEE", "ICALUID")
+);
+
+create table IMIP_INVITATION_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "FROM_ADDR" nvarchar2(255),
+    "TO_ADDR" nvarchar2(255),
+    "ICALENDAR_TEXT" nclob
+);
+
+create table IMIP_POLLING_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table IMIP_REPLY_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "ORGANIZER" nvarchar2(255),
+    "ATTENDEE" nvarchar2(255),
+    "ICALENDAR_TEXT" nclob
+);
+
+create table PUSH_NOTIFICATION_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "PUSH_ID" nvarchar2(255)
+);
+
 create table CALENDARSERVER (
     "NAME" nvarchar2(255) primary key,
     "VALUE" nvarchar2(255)
 );
 
-insert into CALENDARSERVER (NAME, VALUE) values ('VERSION', '15');
+insert into CALENDARSERVER (NAME, VALUE) values ('VERSION', '16');
 insert into CALENDARSERVER (NAME, VALUE) values ('CALENDAR-DATAVERSION', '3');
 insert into CALENDARSERVER (NAME, VALUE) values ('ADDRESSBOOK-DATAVERSION', '1');
-
 create index NOTIFICATION_NOTIFICA_f891f5f9 on NOTIFICATION (
     NOTIFICATION_HOME_RESOURCE_ID
 );
@@ -358,3 +394,7 @@
     RESOURCE_KEY
 );
 
+create index IMIP_TOKENS_TOKEN_e94b918f on IMIP_TOKENS (
+    TOKEN
+);
+

Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql	2013-02-16 00:11:19 UTC (rev 10748)
@@ -492,7 +492,73 @@
 create index APN_SUBSCRIPTIONS_RESOURCE_KEY
    on APN_SUBSCRIPTIONS(RESOURCE_KEY);
 
+-----------------
+-- IMIP Tokens --
+-----------------
 
+create table IMIP_TOKENS (
+  TOKEN                         varchar(255) not null,
+  ORGANIZER                     varchar(255) not null,
+  ATTENDEE                      varchar(255) not null,
+  ICALUID                       varchar(255) not null,
+  ACCESSED                      timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+  primary key (ORGANIZER, ATTENDEE, ICALUID) -- implicit index
+);
+
+create index IMIP_TOKENS_TOKEN
+   on IMIP_TOKENS(TOKEN);
+
+----------------
+-- Work Items --
+----------------
+
+create sequence WORKITEM_SEQ;
+
+---------------------------
+-- IMIP Inivitation Work --
+---------------------------
+
+create table IMIP_INVITATION_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  FROM_ADDR                     varchar(255) not null,
+  TO_ADDR                       varchar(255) not null,
+  ICALENDAR_TEXT                text         not null
+);
+
+-----------------------
+-- IMIP Polling Work --
+-----------------------
+
+create table IMIP_POLLING_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+---------------------
+-- IMIP Reply Work --
+---------------------
+
+create table IMIP_REPLY_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  ORGANIZER                     varchar(255) not null,
+  ATTENDEE                      varchar(255) not null,
+  ICALENDAR_TEXT                text         not null
+);
+
+------------------------
+-- Push Notifications --
+------------------------
+
+create table PUSH_NOTIFICATION_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  PUSH_ID                       varchar(255) not null
+);
+
+
 --------------------
 -- Schema Version --
 --------------------
@@ -502,6 +568,6 @@
   VALUE                         varchar(255)
 );
 
-insert into CALENDARSERVER values ('VERSION', '15');
+insert into CALENDARSERVER values ('VERSION', '16');
 insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '3');
 insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '1');

Added: CalendarServer/trunk/txdav/common/datastore/sql_schema/old/oracle-dialect/v15.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/old/oracle-dialect/v15.sql	                        (rev 0)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/old/oracle-dialect/v15.sql	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,360 @@
+create sequence RESOURCE_ID_SEQ;
+create sequence INSTANCE_ID_SEQ;
+create sequence ATTACHMENT_ID_SEQ;
+create sequence REVISION_SEQ;
+create table NODE_INFO (
+    "HOSTNAME" nvarchar2(255),
+    "PID" integer not null,
+    "PORT" integer not null,
+    "TIME" timestamp default CURRENT_TIMESTAMP at time zone 'UTC' not null, 
+    primary key("HOSTNAME", "PORT")
+);
+
+create table NAMED_LOCK (
+    "LOCK_NAME" nvarchar2(255) primary key
+);
+
+create table CALENDAR_HOME (
+    "RESOURCE_ID" integer primary key,
+    "OWNER_UID" nvarchar2(255) unique,
+    "DATAVERSION" integer default 0 not null
+);
+
+create table CALENDAR_HOME_METADATA (
+    "RESOURCE_ID" integer primary key references CALENDAR_HOME on delete cascade,
+    "QUOTA_USED_BYTES" integer default 0 not null,
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table CALENDAR (
+    "RESOURCE_ID" integer primary key
+);
+
+create table CALENDAR_METADATA (
+    "RESOURCE_ID" integer primary key references CALENDAR on delete cascade,
+    "SUPPORTED_COMPONENTS" nvarchar2(255) default null,
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table NOTIFICATION_HOME (
+    "RESOURCE_ID" integer primary key,
+    "OWNER_UID" nvarchar2(255) unique
+);
+
+create table NOTIFICATION (
+    "RESOURCE_ID" integer primary key,
+    "NOTIFICATION_HOME_RESOURCE_ID" integer not null references NOTIFICATION_HOME,
+    "NOTIFICATION_UID" nvarchar2(255),
+    "XML_TYPE" nvarchar2(255),
+    "XML_DATA" nclob,
+    "MD5" nchar(32),
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC', 
+    unique("NOTIFICATION_UID", "NOTIFICATION_HOME_RESOURCE_ID")
+);
+
+create table CALENDAR_BIND (
+    "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
+    "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
+    "CALENDAR_RESOURCE_NAME" nvarchar2(255) not null,
+    "BIND_MODE" integer not null,
+    "BIND_STATUS" integer not null,
+    "MESSAGE" nclob, 
+    primary key("CALENDAR_HOME_RESOURCE_ID", "CALENDAR_RESOURCE_ID"), 
+    unique("CALENDAR_HOME_RESOURCE_ID", "CALENDAR_RESOURCE_NAME")
+);
+
+create table CALENDAR_BIND_MODE (
+    "ID" integer primary key,
+    "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('own', 0);
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('read', 1);
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('write', 2);
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('direct', 3);
+create table CALENDAR_BIND_STATUS (
+    "ID" integer primary key,
+    "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('invited', 0);
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('accepted', 1);
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('declined', 2);
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('invalid', 3);
+create table CALENDAR_OBJECT (
+    "RESOURCE_ID" integer primary key,
+    "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
+    "RESOURCE_NAME" nvarchar2(255),
+    "ICALENDAR_TEXT" nclob,
+    "ICALENDAR_UID" nvarchar2(255),
+    "ICALENDAR_TYPE" nvarchar2(255),
+    "ATTACHMENTS_MODE" integer default 0 not null,
+    "DROPBOX_ID" nvarchar2(255),
+    "ORGANIZER" nvarchar2(255),
+    "ORGANIZER_OBJECT" integer references CALENDAR_OBJECT,
+    "RECURRANCE_MIN" date,
+    "RECURRANCE_MAX" date,
+    "ACCESS" integer default 0 not null,
+    "SCHEDULE_OBJECT" integer default 0,
+    "SCHEDULE_TAG" nvarchar2(36) default null,
+    "SCHEDULE_ETAGS" nclob default null,
+    "PRIVATE_COMMENTS" integer default 0 not null,
+    "MD5" nchar(32),
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC', 
+    unique("CALENDAR_RESOURCE_ID", "RESOURCE_NAME")
+);
+
+create table CALENDAR_OBJECT_ATTACHMENTS_MO (
+    "ID" integer primary key,
+    "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into CALENDAR_OBJECT_ATTACHMENTS_MO (DESCRIPTION, ID) values ('none', 0);
+insert into CALENDAR_OBJECT_ATTACHMENTS_MO (DESCRIPTION, ID) values ('read', 1);
+insert into CALENDAR_OBJECT_ATTACHMENTS_MO (DESCRIPTION, ID) values ('write', 2);
+create table CALENDAR_ACCESS_TYPE (
+    "ID" integer primary key,
+    "DESCRIPTION" nvarchar2(32) unique
+);
+
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('', 0);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('public', 1);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('private', 2);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('confidential', 3);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('restricted', 4);
+create table TIME_RANGE (
+    "INSTANCE_ID" integer primary key,
+    "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
+    "CALENDAR_OBJECT_RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade,
+    "FLOATING" integer not null,
+    "START_DATE" timestamp not null,
+    "END_DATE" timestamp not null,
+    "FBTYPE" integer not null,
+    "TRANSPARENT" integer not null
+);
+
+create table FREE_BUSY_TYPE (
+    "ID" integer primary key,
+    "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('unknown', 0);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('free', 1);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('busy', 2);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('busy-unavailable', 3);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('busy-tentative', 4);
+create table TRANSPARENCY (
+    "TIME_RANGE_INSTANCE_ID" integer not null references TIME_RANGE on delete cascade,
+    "USER_ID" nvarchar2(255),
+    "TRANSPARENT" integer not null
+);
+
+create table ATTACHMENT (
+    "ATTACHMENT_ID" integer primary key,
+    "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
+    "DROPBOX_ID" nvarchar2(255),
+    "CONTENT_TYPE" nvarchar2(255),
+    "SIZE" integer not null,
+    "MD5" nchar(32),
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "PATH" nvarchar2(1024)
+);
+
+create table ATTACHMENT_CALENDAR_OBJECT (
+    "ATTACHMENT_ID" integer not null references ATTACHMENT on delete cascade,
+    "MANAGED_ID" nvarchar2(255),
+    "CALENDAR_OBJECT_RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade, 
+    primary key("ATTACHMENT_ID", "CALENDAR_OBJECT_RESOURCE_ID"), 
+    unique("MANAGED_ID", "CALENDAR_OBJECT_RESOURCE_ID")
+);
+
+create table RESOURCE_PROPERTY (
+    "RESOURCE_ID" integer not null,
+    "NAME" nvarchar2(255),
+    "VALUE" nclob,
+    "VIEWER_UID" nvarchar2(255), 
+    primary key("RESOURCE_ID", "NAME", "VIEWER_UID")
+);
+
+create table ADDRESSBOOK_HOME (
+    "RESOURCE_ID" integer primary key,
+    "OWNER_UID" nvarchar2(255) unique,
+    "DATAVERSION" integer default 0 not null
+);
+
+create table ADDRESSBOOK_HOME_METADATA (
+    "RESOURCE_ID" integer primary key references ADDRESSBOOK_HOME on delete cascade,
+    "QUOTA_USED_BYTES" integer default 0 not null,
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table ADDRESSBOOK (
+    "RESOURCE_ID" integer primary key
+);
+
+create table ADDRESSBOOK_METADATA (
+    "RESOURCE_ID" integer primary key references ADDRESSBOOK on delete cascade,
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table ADDRESSBOOK_BIND (
+    "ADDRESSBOOK_HOME_RESOURCE_ID" integer not null references ADDRESSBOOK_HOME,
+    "ADDRESSBOOK_RESOURCE_ID" integer not null references ADDRESSBOOK on delete cascade,
+    "ADDRESSBOOK_RESOURCE_NAME" nvarchar2(255) not null,
+    "BIND_MODE" integer not null,
+    "BIND_STATUS" integer not null,
+    "MESSAGE" nclob, 
+    primary key("ADDRESSBOOK_HOME_RESOURCE_ID", "ADDRESSBOOK_RESOURCE_ID"), 
+    unique("ADDRESSBOOK_HOME_RESOURCE_ID", "ADDRESSBOOK_RESOURCE_NAME")
+);
+
+create table ADDRESSBOOK_OBJECT (
+    "RESOURCE_ID" integer primary key,
+    "ADDRESSBOOK_RESOURCE_ID" integer not null references ADDRESSBOOK on delete cascade,
+    "RESOURCE_NAME" nvarchar2(255),
+    "VCARD_TEXT" nclob,
+    "VCARD_UID" nvarchar2(255),
+    "MD5" nchar(32),
+    "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC', 
+    unique("ADDRESSBOOK_RESOURCE_ID", "RESOURCE_NAME"), 
+    unique("ADDRESSBOOK_RESOURCE_ID", "VCARD_UID")
+);
+
+create table CALENDAR_OBJECT_REVISIONS (
+    "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
+    "CALENDAR_RESOURCE_ID" integer references CALENDAR,
+    "CALENDAR_NAME" nvarchar2(255) default null,
+    "RESOURCE_NAME" nvarchar2(255),
+    "REVISION" integer not null,
+    "DELETED" integer not null
+);
+
+create table ADDRESSBOOK_OBJECT_REVISIONS (
+    "ADDRESSBOOK_HOME_RESOURCE_ID" integer not null references ADDRESSBOOK_HOME,
+    "ADDRESSBOOK_RESOURCE_ID" integer references ADDRESSBOOK,
+    "ADDRESSBOOK_NAME" nvarchar2(255) default null,
+    "RESOURCE_NAME" nvarchar2(255),
+    "REVISION" integer not null,
+    "DELETED" integer not null
+);
+
+create table NOTIFICATION_OBJECT_REVISIONS (
+    "NOTIFICATION_HOME_RESOURCE_ID" integer not null references NOTIFICATION_HOME on delete cascade,
+    "RESOURCE_NAME" nvarchar2(255),
+    "REVISION" integer not null,
+    "DELETED" integer not null, 
+    unique("NOTIFICATION_HOME_RESOURCE_ID", "RESOURCE_NAME")
+);
+
+create table APN_SUBSCRIPTIONS (
+    "TOKEN" nvarchar2(255),
+    "RESOURCE_KEY" nvarchar2(255),
+    "MODIFIED" integer not null,
+    "SUBSCRIBER_GUID" nvarchar2(255),
+    "USER_AGENT" nvarchar2(255) default null,
+    "IP_ADDR" nvarchar2(255) default null, 
+    primary key("TOKEN", "RESOURCE_KEY")
+);
+
+create table CALENDARSERVER (
+    "NAME" nvarchar2(255) primary key,
+    "VALUE" nvarchar2(255)
+);
+
+insert into CALENDARSERVER (NAME, VALUE) values ('VERSION', '15');
+insert into CALENDARSERVER (NAME, VALUE) values ('CALENDAR-DATAVERSION', '3');
+insert into CALENDARSERVER (NAME, VALUE) values ('ADDRESSBOOK-DATAVERSION', '1');
+
+create index NOTIFICATION_NOTIFICA_f891f5f9 on NOTIFICATION (
+    NOTIFICATION_HOME_RESOURCE_ID
+);
+
+create index CALENDAR_BIND_RESOURC_e57964d4 on CALENDAR_BIND (
+    CALENDAR_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_CALEN_a9a453a9 on CALENDAR_OBJECT (
+    CALENDAR_RESOURCE_ID,
+    ICALENDAR_UID
+);
+
+create index CALENDAR_OBJECT_CALEN_96e83b73 on CALENDAR_OBJECT (
+    CALENDAR_RESOURCE_ID,
+    RECURRANCE_MAX
+);
+
+create index CALENDAR_OBJECT_ORGAN_7ce24750 on CALENDAR_OBJECT (
+    ORGANIZER_OBJECT
+);
+
+create index CALENDAR_OBJECT_DROPB_de041d80 on CALENDAR_OBJECT (
+    DROPBOX_ID
+);
+
+create index TIME_RANGE_CALENDAR_R_beb6e7eb on TIME_RANGE (
+    CALENDAR_RESOURCE_ID
+);
+
+create index TIME_RANGE_CALENDAR_O_acf37bd1 on TIME_RANGE (
+    CALENDAR_OBJECT_RESOURCE_ID
+);
+
+create index TRANSPARENCY_TIME_RAN_5f34467f on TRANSPARENCY (
+    TIME_RANGE_INSTANCE_ID
+);
+
+create index ATTACHMENT_CALENDAR_H_0078845c on ATTACHMENT (
+    CALENDAR_HOME_RESOURCE_ID
+);
+
+create index ADDRESSBOOK_BIND_RESO_205aa75c on ADDRESSBOOK_BIND (
+    ADDRESSBOOK_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_REVIS_3a3956c4 on CALENDAR_OBJECT_REVISIONS (
+    CALENDAR_HOME_RESOURCE_ID,
+    CALENDAR_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_REVIS_2643d556 on CALENDAR_OBJECT_REVISIONS (
+    CALENDAR_RESOURCE_ID,
+    RESOURCE_NAME
+);
+
+create index CALENDAR_OBJECT_REVIS_265c8acf on CALENDAR_OBJECT_REVISIONS (
+    CALENDAR_RESOURCE_ID,
+    REVISION
+);
+
+create index ADDRESSBOOK_OBJECT_RE_f460d62d on ADDRESSBOOK_OBJECT_REVISIONS (
+    ADDRESSBOOK_HOME_RESOURCE_ID,
+    ADDRESSBOOK_RESOURCE_ID
+);
+
+create index ADDRESSBOOK_OBJECT_RE_9a848f39 on ADDRESSBOOK_OBJECT_REVISIONS (
+    ADDRESSBOOK_RESOURCE_ID,
+    RESOURCE_NAME
+);
+
+create index ADDRESSBOOK_OBJECT_RE_cb101e6b on ADDRESSBOOK_OBJECT_REVISIONS (
+    ADDRESSBOOK_RESOURCE_ID,
+    REVISION
+);
+
+create index NOTIFICATION_OBJECT_R_036a9cee on NOTIFICATION_OBJECT_REVISIONS (
+    NOTIFICATION_HOME_RESOURCE_ID,
+    REVISION
+);
+
+create index APN_SUBSCRIPTIONS_RES_9610d78e on APN_SUBSCRIPTIONS (
+    RESOURCE_KEY
+);
+

Added: CalendarServer/trunk/txdav/common/datastore/sql_schema/old/postgres-dialect/v15.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/old/postgres-dialect/v15.sql	                        (rev 0)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/old/postgres-dialect/v15.sql	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,507 @@
+-- -*- test-case-name: txdav.caldav.datastore.test.test_sql,txdav.carddav.datastore.test.test_sql -*-
+
+----
+-- Copyright (c) 2010-2013 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-----------------
+-- Resource ID --
+-----------------
+
+create sequence RESOURCE_ID_SEQ;
+
+-------------------------
+-- Cluster Bookkeeping --
+-------------------------
+
+-- Information about a process connected to this database.
+
+-- Note that this must match the node info schema in twext.enterprise.queue.
+create table NODE_INFO (
+  HOSTNAME  varchar(255) not null,
+  PID       integer not null,
+  PORT      integer not null,
+  TIME      timestamp not null default timezone('UTC', CURRENT_TIMESTAMP),
+
+  primary key (HOSTNAME, PORT)
+);
+
+-- Unique named locks.  This table should always be empty, but rows are
+-- temporarily created in order to prevent undesirable concurrency.
+create table NAMED_LOCK (
+    LOCK_NAME varchar(255) primary key
+);
+
+
+-------------------
+-- Calendar Home --
+-------------------
+
+create table CALENDAR_HOME (
+  RESOURCE_ID      integer      primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+  OWNER_UID        varchar(255) not null unique,                                 -- implicit index
+  DATAVERSION      integer      default 0 not null
+);
+
+----------------------------
+-- Calendar Home Metadata --
+----------------------------
+
+create table CALENDAR_HOME_METADATA (
+  RESOURCE_ID      integer      primary key references CALENDAR_HOME on delete cascade, -- implicit index
+  QUOTA_USED_BYTES integer      default 0 not null,
+  CREATED          timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED         timestamp    default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+--------------
+-- Calendar --
+--------------
+
+create table CALENDAR (
+  RESOURCE_ID integer   primary key default nextval('RESOURCE_ID_SEQ') -- implicit index
+);
+
+
+-----------------------
+-- Calendar Metadata --
+-----------------------
+
+create table CALENDAR_METADATA (
+  RESOURCE_ID           integer   primary key references CALENDAR on delete cascade, -- implicit index
+  SUPPORTED_COMPONENTS  varchar(255) default null,
+  CREATED               timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED              timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+
+---------------------------
+-- Sharing Notifications --
+---------------------------
+
+create table NOTIFICATION_HOME (
+  RESOURCE_ID integer      primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+  OWNER_UID   varchar(255) not null unique                                 -- implicit index
+);
+
+create table NOTIFICATION (
+  RESOURCE_ID                   integer      primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+  NOTIFICATION_HOME_RESOURCE_ID integer      not null references NOTIFICATION_HOME,
+  NOTIFICATION_UID              varchar(255) not null,
+  XML_TYPE                      varchar(255) not null,
+  XML_DATA                      text         not null,
+  MD5                           char(32)     not null,
+  CREATED                       timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED                      timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+  unique(NOTIFICATION_UID, NOTIFICATION_HOME_RESOURCE_ID) -- implicit index
+);
+
+create index NOTIFICATION_NOTIFICATION_HOME_RESOURCE_ID on
+  NOTIFICATION(NOTIFICATION_HOME_RESOURCE_ID);
+
+-------------------
+-- Calendar Bind --
+-------------------
+
+-- Joins CALENDAR_HOME and CALENDAR
+
+create table CALENDAR_BIND (
+  CALENDAR_HOME_RESOURCE_ID integer      not null references CALENDAR_HOME,
+  CALENDAR_RESOURCE_ID      integer      not null references CALENDAR on delete cascade,
+  CALENDAR_RESOURCE_NAME    varchar(255) not null,
+  BIND_MODE                 integer      not null, -- enum CALENDAR_BIND_MODE
+  BIND_STATUS               integer      not null, -- enum CALENDAR_BIND_STATUS
+  MESSAGE                   text,
+
+  primary key(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_ID), -- implicit index
+  unique(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_NAME)     -- implicit index
+);
+
+create index CALENDAR_BIND_RESOURCE_ID on CALENDAR_BIND(CALENDAR_RESOURCE_ID);
+
+-- Enumeration of calendar bind modes
+
+create table CALENDAR_BIND_MODE (
+  ID          integer     primary key,
+  DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_BIND_MODE values (0, 'own'  );
+insert into CALENDAR_BIND_MODE values (1, 'read' );
+insert into CALENDAR_BIND_MODE values (2, 'write');
+insert into CALENDAR_BIND_MODE values (3, 'direct');
+
+-- Enumeration of statuses
+
+create table CALENDAR_BIND_STATUS (
+  ID          integer     primary key,
+  DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_BIND_STATUS values (0, 'invited' );
+insert into CALENDAR_BIND_STATUS values (1, 'accepted');
+insert into CALENDAR_BIND_STATUS values (2, 'declined');
+insert into CALENDAR_BIND_STATUS values (3, 'invalid');
+
+
+---------------------
+-- Calendar Object --
+---------------------
+
+create table CALENDAR_OBJECT (
+  RESOURCE_ID          integer      primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+  CALENDAR_RESOURCE_ID integer      not null references CALENDAR on delete cascade,
+  RESOURCE_NAME        varchar(255) not null,
+  ICALENDAR_TEXT       text         not null,
+  ICALENDAR_UID        varchar(255) not null,
+  ICALENDAR_TYPE       varchar(255) not null,
+  ATTACHMENTS_MODE     integer      default 0 not null, -- enum CALENDAR_OBJECT_ATTACHMENTS_MODE
+  DROPBOX_ID           varchar(255),
+  ORGANIZER            varchar(255),
+  ORGANIZER_OBJECT     integer      references CALENDAR_OBJECT,
+  RECURRANCE_MIN       date,        -- minimum date that recurrences have been expanded to.
+  RECURRANCE_MAX       date,        -- maximum date that recurrences have been expanded to.
+  ACCESS               integer      default 0 not null,
+  SCHEDULE_OBJECT      boolean      default false,
+  SCHEDULE_TAG         varchar(36)  default null,
+  SCHEDULE_ETAGS       text         default null,
+  PRIVATE_COMMENTS     boolean      default false not null,
+  MD5                  char(32)     not null,
+  CREATED              timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED             timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+
+  unique (CALENDAR_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+
+  -- since the 'inbox' is a 'calendar resource' for the purpose of storing
+  -- calendar objects, this constraint has to be selectively enforced by the
+  -- application layer.
+
+  -- unique(CALENDAR_RESOURCE_ID, ICALENDAR_UID)
+);
+
+create index CALENDAR_OBJECT_CALENDAR_RESOURCE_ID_AND_ICALENDAR_UID on
+  CALENDAR_OBJECT(CALENDAR_RESOURCE_ID, ICALENDAR_UID);
+
+create index CALENDAR_OBJECT_CALENDAR_RESOURCE_ID_RECURRANCE_MAX on
+  CALENDAR_OBJECT(CALENDAR_RESOURCE_ID, RECURRANCE_MAX);
+
+create index CALENDAR_OBJECT_ORGANIZER_OBJECT on
+  CALENDAR_OBJECT(ORGANIZER_OBJECT);
+
+create index CALENDAR_OBJECT_DROPBOX_ID on
+  CALENDAR_OBJECT(DROPBOX_ID);
+
+-- Enumeration of attachment modes
+
+create table CALENDAR_OBJECT_ATTACHMENTS_MODE (
+  ID          integer     primary key,
+  DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (0, 'none' );
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (1, 'read' );
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (2, 'write');
+
+
+-- Enumeration of calendar access types
+
+create table CALENDAR_ACCESS_TYPE (
+  ID          integer     primary key,
+  DESCRIPTION varchar(32) not null unique
+);
+
+insert into CALENDAR_ACCESS_TYPE values (0, ''             );
+insert into CALENDAR_ACCESS_TYPE values (1, 'public'       );
+insert into CALENDAR_ACCESS_TYPE values (2, 'private'      );
+insert into CALENDAR_ACCESS_TYPE values (3, 'confidential' );
+insert into CALENDAR_ACCESS_TYPE values (4, 'restricted'   );
+
+-----------------
+-- Instance ID --
+-----------------
+
+create sequence INSTANCE_ID_SEQ;
+
+
+----------------
+-- Time Range --
+----------------
+
+create table TIME_RANGE (
+  INSTANCE_ID                 integer        primary key default nextval('INSTANCE_ID_SEQ'), -- implicit index
+  CALENDAR_RESOURCE_ID        integer        not null references CALENDAR on delete cascade,
+  CALENDAR_OBJECT_RESOURCE_ID integer        not null references CALENDAR_OBJECT on delete cascade,
+  FLOATING                    boolean        not null,
+  START_DATE                  timestamp      not null,
+  END_DATE                    timestamp      not null,
+  FBTYPE                      integer        not null,
+  TRANSPARENT                 boolean        not null
+);
+
+create index TIME_RANGE_CALENDAR_RESOURCE_ID on
+  TIME_RANGE(CALENDAR_RESOURCE_ID);
+create index TIME_RANGE_CALENDAR_OBJECT_RESOURCE_ID on
+  TIME_RANGE(CALENDAR_OBJECT_RESOURCE_ID);
+
+
+-- Enumeration of free/busy types
+
+create table FREE_BUSY_TYPE (
+  ID          integer     primary key,
+  DESCRIPTION varchar(16) not null unique
+);
+
+insert into FREE_BUSY_TYPE values (0, 'unknown'         );
+insert into FREE_BUSY_TYPE values (1, 'free'            );
+insert into FREE_BUSY_TYPE values (2, 'busy'            );
+insert into FREE_BUSY_TYPE values (3, 'busy-unavailable');
+insert into FREE_BUSY_TYPE values (4, 'busy-tentative'  );
+
+
+------------------
+-- Transparency --
+------------------
+
+create table TRANSPARENCY (
+  TIME_RANGE_INSTANCE_ID      integer      not null references TIME_RANGE on delete cascade,
+  USER_ID                     varchar(255) not null,
+  TRANSPARENT                 boolean      not null
+);
+
+create index TRANSPARENCY_TIME_RANGE_INSTANCE_ID on
+  TRANSPARENCY(TIME_RANGE_INSTANCE_ID);
+
+
+----------------
+-- Attachment --
+----------------
+
+create sequence ATTACHMENT_ID_SEQ;
+
+create table ATTACHMENT (
+  ATTACHMENT_ID               integer           primary key default nextval('ATTACHMENT_ID_SEQ'), -- implicit index
+  CALENDAR_HOME_RESOURCE_ID   integer           not null references CALENDAR_HOME,
+  DROPBOX_ID                  varchar(255),
+  CONTENT_TYPE                varchar(255)      not null,
+  SIZE                        integer           not null,
+  MD5                         char(32)          not null,
+  CREATED                     timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED                    timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+  PATH                        varchar(1024)     not null
+);
+
+create index ATTACHMENT_CALENDAR_HOME_RESOURCE_ID on
+  ATTACHMENT(CALENDAR_HOME_RESOURCE_ID);
+
+-- Many-to-many relationship between attachments and calendar objects
+create table ATTACHMENT_CALENDAR_OBJECT (
+  ATTACHMENT_ID                  integer      not null references ATTACHMENT on delete cascade,
+  MANAGED_ID                     varchar(255) not null,
+  CALENDAR_OBJECT_RESOURCE_ID    integer      not null references CALENDAR_OBJECT on delete cascade,
+
+  primary key (ATTACHMENT_ID, CALENDAR_OBJECT_RESOURCE_ID), -- implicit index
+  unique (MANAGED_ID, CALENDAR_OBJECT_RESOURCE_ID) --implicit index
+);
+
+
+-----------------------
+-- Resource Property --
+-----------------------
+
+create table RESOURCE_PROPERTY (
+  RESOURCE_ID integer      not null, -- foreign key: *.RESOURCE_ID
+  NAME        varchar(255) not null,
+  VALUE       text         not null, -- FIXME: xml?
+  VIEWER_UID  varchar(255),
+
+  primary key (RESOURCE_ID, NAME, VIEWER_UID) -- implicit index
+);
+
+
+----------------------
+-- AddressBook Home --
+----------------------
+
+create table ADDRESSBOOK_HOME (
+  RESOURCE_ID      integer      primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+  OWNER_UID        varchar(255) not null unique,                                -- implicit index
+  DATAVERSION      integer      default 0 not null
+);
+
+-------------------------------
+-- AddressBook Home Metadata --
+-------------------------------
+
+create table ADDRESSBOOK_HOME_METADATA (
+  RESOURCE_ID      integer      primary key references ADDRESSBOOK_HOME on delete cascade, -- implicit index
+  QUOTA_USED_BYTES integer      default 0 not null,
+  CREATED          timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED         timestamp    default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+-----------------
+-- AddressBook --
+-----------------
+
+create table ADDRESSBOOK (
+  RESOURCE_ID integer   primary key default nextval('RESOURCE_ID_SEQ') -- implicit index
+);
+
+
+--------------------------
+-- AddressBook Metadata --
+--------------------------
+
+create table ADDRESSBOOK_METADATA (
+  RESOURCE_ID integer   primary key references ADDRESSBOOK on delete cascade, -- implicit index
+  CREATED     timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED    timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+
+----------------------
+-- AddressBook Bind --
+----------------------
+
+-- Joins ADDRESSBOOK_HOME and ADDRESSBOOK
+
+create table ADDRESSBOOK_BIND (
+  ADDRESSBOOK_HOME_RESOURCE_ID integer      not null references ADDRESSBOOK_HOME,
+  ADDRESSBOOK_RESOURCE_ID      integer      not null references ADDRESSBOOK on delete cascade,
+  ADDRESSBOOK_RESOURCE_NAME    varchar(255) not null,
+  BIND_MODE                    integer      not null, -- enum CALENDAR_BIND_MODE
+  BIND_STATUS                  integer      not null, -- enum CALENDAR_BIND_STATUS
+  MESSAGE                      text,                  -- FIXME: xml?
+
+  primary key (ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_ID), -- implicit index
+  unique (ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_NAME)     -- implicit index
+);
+
+create index ADDRESSBOOK_BIND_RESOURCE_ID on
+  ADDRESSBOOK_BIND(ADDRESSBOOK_RESOURCE_ID);
+
+create table ADDRESSBOOK_OBJECT (
+  RESOURCE_ID             integer      primary key default nextval('RESOURCE_ID_SEQ'),    -- implicit index
+  ADDRESSBOOK_RESOURCE_ID integer      not null references ADDRESSBOOK on delete cascade,
+  RESOURCE_NAME           varchar(255) not null,
+  VCARD_TEXT              text         not null,
+  VCARD_UID               varchar(255) not null,
+  MD5                     char(32)     not null,
+  CREATED                 timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  MODIFIED                timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+
+  unique (ADDRESSBOOK_RESOURCE_ID, RESOURCE_NAME), -- implicit index
+  unique (ADDRESSBOOK_RESOURCE_ID, VCARD_UID)      -- implicit index
+);
+
+---------------
+-- Revisions --
+---------------
+
+create sequence REVISION_SEQ;
+
+
+---------------
+-- Revisions --
+---------------
+
+create table CALENDAR_OBJECT_REVISIONS (
+  CALENDAR_HOME_RESOURCE_ID integer      not null references CALENDAR_HOME,
+  CALENDAR_RESOURCE_ID      integer      references CALENDAR,
+  CALENDAR_NAME             varchar(255) default null,
+  RESOURCE_NAME             varchar(255),
+  REVISION                  integer      default nextval('REVISION_SEQ') not null,
+  DELETED                   boolean      not null
+);
+
+create index CALENDAR_OBJECT_REVISIONS_HOME_RESOURCE_ID_CALENDAR_RESOURCE_ID
+  on CALENDAR_OBJECT_REVISIONS(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_ID);
+
+create index CALENDAR_OBJECT_REVISIONS_RESOURCE_ID_RESOURCE_NAME
+  on CALENDAR_OBJECT_REVISIONS(CALENDAR_RESOURCE_ID, RESOURCE_NAME);
+
+create index CALENDAR_OBJECT_REVISIONS_RESOURCE_ID_REVISION
+  on CALENDAR_OBJECT_REVISIONS(CALENDAR_RESOURCE_ID, REVISION);
+
+-------------------------------
+-- AddressBook Object Revisions --
+-------------------------------
+
+create table ADDRESSBOOK_OBJECT_REVISIONS (
+  ADDRESSBOOK_HOME_RESOURCE_ID integer      not null references ADDRESSBOOK_HOME,
+  ADDRESSBOOK_RESOURCE_ID      integer      references ADDRESSBOOK,
+  ADDRESSBOOK_NAME             varchar(255) default null,
+  RESOURCE_NAME                varchar(255),
+  REVISION                     integer      default nextval('REVISION_SEQ') not null,
+  DELETED                      boolean      not null
+);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_HOME_RESOURCE_ID_ADDRESSBOOK_RESOURCE_ID
+  on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_ID);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_RESOURCE_ID_RESOURCE_NAME
+  on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_RESOURCE_ID, RESOURCE_NAME);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_RESOURCE_ID_REVISION
+  on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_RESOURCE_ID, REVISION);
+
+-----------------------------------
+-- Notification Object Revisions --
+-----------------------------------
+
+create table NOTIFICATION_OBJECT_REVISIONS (
+  NOTIFICATION_HOME_RESOURCE_ID integer      not null references NOTIFICATION_HOME on delete cascade,
+  RESOURCE_NAME                 varchar(255),
+  REVISION                      integer      default nextval('REVISION_SEQ') not null,
+  DELETED                       boolean      not null,
+
+  unique(NOTIFICATION_HOME_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+);
+
+create index NOTIFICATION_OBJECT_REVISIONS_RESOURCE_ID_REVISION
+  on NOTIFICATION_OBJECT_REVISIONS(NOTIFICATION_HOME_RESOURCE_ID, REVISION);
+
+-------------------------------------------
+-- Apple Push Notification Subscriptions --
+-------------------------------------------
+
+create table APN_SUBSCRIPTIONS (
+  TOKEN                         varchar(255) not null,
+  RESOURCE_KEY                  varchar(255) not null,
+  MODIFIED                      integer not null,
+  SUBSCRIBER_GUID               varchar(255) not null,
+  USER_AGENT                    varchar(255) default null,
+  IP_ADDR                       varchar(255) default null,
+
+  primary key (TOKEN, RESOURCE_KEY) -- implicit index
+);
+
+create index APN_SUBSCRIPTIONS_RESOURCE_KEY
+   on APN_SUBSCRIPTIONS(RESOURCE_KEY);
+
+
+--------------------
+-- Schema Version --
+--------------------
+
+create table CALENDARSERVER (
+  NAME                          varchar(255) primary key, -- implicit index
+  VALUE                         varchar(255)
+);
+
+insert into CALENDARSERVER values ('VERSION', '15');
+insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '3');
+insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '1');

Added: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_15_to_16.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_15_to_16.sql	                        (rev 0)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_15_to_16.sql	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,63 @@
+----
+-- Copyright (c) 2011-2013 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-------------------------------------------------
+-- Upgrade database schema from VERSION 15 to 16 --
+-------------------------------------------------
+
+
+create sequence WORKITEM_SEQ;
+
+create table IMIP_TOKENS (
+    "TOKEN" nvarchar2(255),
+    "ORGANIZER" nvarchar2(255),
+    "ATTENDEE" nvarchar2(255),
+    "ICALUID" nvarchar2(255),
+    "ACCESSED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    primary key("ORGANIZER", "ATTENDEE", "ICALUID")
+);
+
+create table IMIP_INVITATION_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "FROM_ADDR" nvarchar2(255),
+    "TO_ADDR" nvarchar2(255),
+    "ICALENDAR_TEXT" nclob
+);
+
+create table IMIP_POLLING_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table IMIP_REPLY_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "ORGANIZER" nvarchar2(255),
+    "ATTENDEE" nvarchar2(255),
+    "ICALENDAR_TEXT" nclob
+);
+
+create table PUSH_NOTIFICATION_WORK (
+    "WORK_ID" integer primary key not null,
+    "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+    "PUSH_ID" nvarchar2(255)
+);
+
+
+-- Now update the version
+update CALENDARSERVER set VALUE = '16' where NAME = 'VERSION';
+

Added: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_15_to_16.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_15_to_16.sql	                        (rev 0)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_15_to_16.sql	2013-02-16 00:11:19 UTC (rev 10748)
@@ -0,0 +1,91 @@
+----
+-- Copyright (c) 2011-2013 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-------------------------------------------------
+-- Upgrade database schema from VERSION 15 to 16 --
+-------------------------------------------------
+
+
+-----------------
+-- IMIP Tokens --
+-----------------
+
+create table IMIP_TOKENS (
+  TOKEN                         varchar(255) not null,
+  ORGANIZER                     varchar(255) not null,
+  ATTENDEE                      varchar(255) not null,
+  ICALUID                       varchar(255) not null,
+  ACCESSED                      timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+  primary key (ORGANIZER, ATTENDEE, ICALUID) -- implicit index
+);
+
+create index IMIP_TOKENS_TOKEN
+   on IMIP_TOKENS(TOKEN);
+
+----------------
+-- Work Items --
+----------------
+
+create sequence WORKITEM_SEQ;
+
+---------------------------
+-- IMIP Inivitation Work --
+---------------------------
+
+create table IMIP_INVITATION_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  FROM_ADDR                     varchar(255) not null,
+  TO_ADDR                       varchar(255) not null,
+  ICALENDAR_TEXT                text         not null
+);
+
+-----------------------
+-- IMIP Polling Work --
+-----------------------
+
+create table IMIP_POLLING_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+---------------------
+-- IMIP Reply Work --
+---------------------
+
+create table IMIP_REPLY_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  ORGANIZER                     varchar(255) not null,
+  ATTENDEE                      varchar(255) not null,
+  ICALENDAR_TEXT                text         not null
+);
+
+------------------------
+-- Push Notifications --
+------------------------
+
+create table PUSH_NOTIFICATION_WORK (
+  WORK_ID                       integer primary key default nextval('WORKITEM_SEQ') not null,
+  NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
+  PUSH_ID                       varchar(255) not null
+);
+
+
+-- Now update the version
+update CALENDARSERVER set VALUE = '16' where NAME = 'VERSION';
+

Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -30,7 +30,7 @@
 from twext.web2.dav.resource import TwistedGETContentMD5
 
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.internet.task import deferLater
 from twisted.python import log
 from twisted.application.service import Service
@@ -45,7 +45,7 @@
 from twext.enterprise.adbapi2 import ConnectionPool
 from twisted.trial.unittest import TestCase
 from twisted.internet.defer import returnValue
-from twistedcaldav.notify import Notifier, NodeCreationException
+from calendarserver.push.notifier import Notifier
 from twext.enterprise.ienterprise import AlreadyFinishedError
 from twistedcaldav.vcard import Component as ABComponent
 
@@ -646,16 +646,7 @@
 
 
 
-class StubNodeCacher(object):
 
-    def waitForNode(self, notifier, nodeName):
-        if "fail" in nodeName:
-            raise NodeCreationException("Could not create node")
-        else:
-            return succeed(True)
-
-
-
 class StubNotifierFactory(object):
     """
     For testing push notifications without an XMPP server.
@@ -663,23 +654,33 @@
 
     def __init__(self):
         self.reset()
-        self.nodeCacher = StubNodeCacher()
-        self.pubSubConfig = {
-            "enabled" : True,
-            "service" : "pubsub.example.com",
-            "host" : "example.com",
-            "port" : "123",
-        }
+        self.hostname = "example.com"
 
 
     def newNotifier(self, label="default", id=None, prefix=None):
         return Notifier(self, label=label, id=id, prefix=prefix)
 
 
-    def send(self, op, id):
-        self.history.append((op, id))
+    def pushKeyForId(self, id):
+        path = "/"
 
+        try:
+            prefix, id = id.split("|", 1)
+            path += "%s/" % (prefix,)
+        except ValueError:
+            # id has no prefix
+            pass
 
+        path += "%s/" % (self.hostname,)
+        if id:
+            path += "%s/" % (id,)
+        return path
+
+
+    def send(self, id):
+        self.history.append(id)
+
+
     def reset(self):
         self.history = []
 

Modified: CalendarServer/trunk/txdav/common/icommondatastore.py
===================================================================
--- CalendarServer/trunk/txdav/common/icommondatastore.py	2013-02-15 23:25:56 UTC (rev 10747)
+++ CalendarServer/trunk/txdav/common/icommondatastore.py	2013-02-16 00:11:19 UTC (rev 10748)
@@ -148,6 +148,13 @@
     Invalid APN subscription values passed in.
     """
 
+# IMIP Tokens
+
+class InvalidIMIPTokenValues(ValueError):
+    """
+    Invalid IMIP token values passed in.
+    """
+
 #
 # Interfaces
 #
@@ -240,7 +247,59 @@
         @return: tuples of (token, key, timestamp, userAgent, ipAddr)
         """
 
+    def imipCreateToken(organizer, attendee, icaluid, token=None):
+        """
+        Add an entry in the database; if no token is provided, one will be
+        generated.
 
+        @param organizer: the CUA of the organizer
+        @type organizer: C{str}
+        @param attendee: the mailto: CUA of the attendee
+        @type organizer: C{str}
+        @param icaluid: the icalendar UID of the VEVENT
+        @type organizer: C{str}
+        @param token: the value to use in the "plus address" of the reply-to
+        @type token: C{str}
+        """
+
+    def imipLookupByToken(token):
+        """
+        Returns the organizer, attendee, and icaluid corresponding to the token
+
+        @param token: the token to look up
+        @type token: C{str}
+        """
+
+
+    def imipGetToken(organizer, attendee, icaluid):
+        """
+        Returns the token (if any) corresponding to the given organizer, attendee,
+        and icaluid combination
+
+        @param organizer: the CUA of the organizer
+        @type organizer: C{str}
+        @param attendee: the mailto: CUA of the attendee
+        @type organizer: C{str}
+        @param icaluid: the icalendar UID of the VEVENT
+        @type organizer: C{str}
+        """
+
+
+    def imipRemoveToken(token):
+        """
+        Removes the entry for the given token.
+
+        @param token: the token to remove
+        @type token: C{str}
+        """
+
+
+    def purgeOldIMIPTokens(olderThan):
+        """
+        Removes all tokens whose access time is before olderThan
+        """
+        
+
 class IShareableCollection(Interface):
     """
     A collection resource which may be shared.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130215/a3129ae8/attachment-0001.html>


More information about the calendarserver-changes mailing list