[CalendarServer-changes] [8148] CalendarServer/trunk/twistedcaldav/timezonestdservice.py

[source_changes at macosforge.org]

Revision: 8148
          http://trac.macosforge.org/projects/calendarserver/changeset/8148
Author:   cdaboo at apple.com
Date:     2011-10-06 03:46:29 -0700 (Thu, 06 Oct 2011)
Log Message:
-----------
Added some debug logging. Fixed date format issue. Do secondary fetching in batches.

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/timezonestdservice.py

Modified: CalendarServer/trunk/twistedcaldav/timezonestdservice.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/timezonestdservice.py	2011-10-05 22:02:03 UTC (rev 8147)
+++ CalendarServer/trunk/twistedcaldav/timezonestdservice.py	2011-10-06 10:46:29 UTC (rev 8148)
@@ -37,7 +37,8 @@
 from twext.web2.http_headers import MimeType
 from twext.web2.stream import MemoryStream
 
-from twisted.internet.defer import succeed, inlineCallbacks, returnValue
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue,\
+    DeferredList
 
 from twistedcaldav import timezonexml, xmlutil
 from twistedcaldav.client.geturl import getURL
@@ -582,7 +583,7 @@
         Create a new DB xml file from scratch by scanning zoneinfo.
         """
 
-        self.dtstamp = PyCalendarDateTime.getNowUTC().getText()
+        self.dtstamp = PyCalendarDateTime.getNowUTC().getXMLText()
         self._scanTZs("")
         self._dumpTZs()
 
@@ -635,7 +636,7 @@
         """
         Update existing DB info by comparing md5's.
         """
-        self.dtstamp = PyCalendarDateTime.getNowUTC().getText()
+        self.dtstamp = PyCalendarDateTime.getNowUTC().getXMLText()
         self.changeCount = 0
         self.changed = set()
         self._scanTZs("", checkIfChanged=True)
@@ -652,7 +653,9 @@
         self.uri = uri
         self.discovered = False
         self._url = None
-        
+
+        log.debug("Configuring secondary server with basepath: %s" % (self.basepath,))
+
         if not os.path.exists(self.basepath):
             os.makedirs(self.basepath)
             
@@ -671,9 +674,11 @@
         Sync local data with that from the server we are replicating.
         """
         
+        log.debug("Sync'ing with secondary server")
         result = (yield self._getTimezoneListFromServer())
         if result is None:
             # Nothing changed since last sync
+            log.debug("No changes on secondary server")
             returnValue(None)
         newdtstamp, newtimezones = result
         
@@ -688,13 +693,21 @@
             if self.timezones[tzid].dtstamp < newtimezones[tzid].dtstamp:
                 changedtzids.add(tzid)
         
-        # Now apply changes
-        for tzid in itertools.chain(newtzids, changedtzids):
-            yield self._getTimezoneFromServer(newtimezones[tzid])
-            
+        log.debug("Fetching %d new, %d changed timezones on secondary server" % (len(newtzids), len(changedtzids),))
+
+        # Now apply changes - do requests in parallel for speedier fetching
+        BATCH = 5
+        tzids = list(itertools.chain(newtzids, changedtzids))
+        tzids.sort()
+        while tzids:
+            yield DeferredList([self._getTimezoneFromServer(newtimezones[tzid]) for tzid in tzids[0:BATCH]])
+            tzids = tzids[BATCH:]
+
         self.dtstamp = newdtstamp
         self._dumpTZs()
         self._buildAliases()
+
+        log.debug("Sync with secondary server complete")
         
         returnValue((len(newtzids), len(changedtzids),))
         
@@ -706,21 +719,24 @@
         
         if self.uri is None:
             if config.TimezoneService.SecondaryService.Host:
-                self.uri = "https://%s/.well-known/timezone" % (self.config.TimezoneService.SecondaryService.Host,)
+                self.uri = "https://%s/.well-known/timezone" % (config.TimezoneService.SecondaryService.Host,)
             elif config.TimezoneService.SecondaryService.URI:
                 self.uri = config.TimezoneService.SecondaryService.URI
         elif not self.uri.startswith("https:") and not self.uri.startswith("http:"):
             self.uri = "https://%s/.well-known/timezone" % (self.uri,)
             
         testURI = "%s?action=capabilities" % (self.uri,)
+        log.debug("Discovering secondary server: %s" % (testURI,))
         response = (yield getURL(testURI))
         if response is None or response.code / 100 != 2:
+            log.error("Unable to discover secondary server: %s" % (testURI,))
             self.discovered = False
             returnValue(False)
         
         # Cache the redirect target
         if hasattr(response, "location"):
             self.uri = response.location 
+            log.debug("Redirected secondary server to: %s" % (self.uri,))
 
         # TODO: Ignoring the data from capabilities for now
 
@@ -743,6 +759,7 @@
         url = "%s?action=list" % (self.uri,)
         if self.dtstamp:
             url = "%s&changedsince=%s" % (url, self.dtstamp,)
+        log.debug("Getting timezone list from secondary server: %s" % (url,))
         response = (yield getURL(url))
         if response is None or response.code / 100 != 2:
             returnValue(None)
@@ -761,13 +778,17 @@
             lastmod = summary.findtext(timezonexml.LastModified.sname())
             aliases = tuple([alias_node.text for alias_node in summary.findall(timezonexml.Alias.sname())])
             timezones[tzid] = TimezoneInfo(tzid, aliases, lastmod, None)
+
+        log.debug("Got %s timezones from secondary server" % (len(timezones),))
         
         returnValue((dtstamp, timezones,))
 
     @inlineCallbacks
     def _getTimezoneFromServer(self, tzinfo):
         # List all from the server
-        response = (yield getURL("%s?action=get&tzid=%s" % (self.uri, tzinfo.tzid,)))
+        url = "%s?action=get&tzid=%s" % (self.uri, tzinfo.tzid,)
+        log.debug("Getting timezone from secondary server: %s" % (url,))
+        response = (yield getURL(url))
         if response is None or response.code / 100 != 2:
             returnValue(None)
         
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20111006/aa21a275/attachment.html>