[CalendarServer-changes] [7518] CalendarServer/branches/users/cdaboo/timezones/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Mon May 23 17:38:33 PDT 2011
Revision: 7518
http://trac.macosforge.org/projects/calendarserver/changeset/7518
Author: cdaboo at apple.com
Date: 2011-05-23 17:38:32 -0700 (Mon, 23 May 2011)
Log Message:
-----------
Secondary timezone service that can pull data from another.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/timezonestdservice.py
CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/xmlutil.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/client/geturl.py
Added: CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/client/geturl.py
===================================================================
--- CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/client/geturl.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/client/geturl.py 2011-05-24 00:38:32 UTC (rev 7518)
@@ -0,0 +1,111 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.python.log import Logger
+
+from twisted.internet import reactor, protocol
+from twisted.internet.defer import inlineCallbacks, Deferred, returnValue
+from twisted.web import http_headers
+from twisted.web.client import Agent
+from twisted.web.http import MOVED_PERMANENTLY, TEMPORARY_REDIRECT, FOUND
+
+from urlparse import urlparse
+from urlparse import urlunparse
+
+__all__ = [
+ "getURL",
+]
+
+log = Logger()
+
+class AccumulatingProtocol(protocol.Protocol):
+ """
+ L{AccumulatingProtocol} is an L{IProtocol} implementation which collects
+ the data delivered to it and can fire a Deferred when it is connected or
+ disconnected.
+
+ @ivar made: A flag indicating whether C{connectionMade} has been called.
+ @ivar data: A string giving all the data passed to C{dataReceived}.
+ @ivar closed: A flag indicated whether C{connectionLost} has been called.
+ @ivar closedReason: The value of the I{reason} parameter passed to
+ C{connectionLost}.
+ @ivar closedDeferred: If set to a L{Deferred}, this will be fired when
+ C{connectionLost} is called.
+ """
+ made = closed = 0
+ closedReason = None
+
+ closedDeferred = None
+
+ data = ""
+
+ factory = None
+
+ def connectionMade(self):
+ self.made = 1
+ if (self.factory is not None and
+ self.factory.protocolConnectionMade is not None):
+ d = self.factory.protocolConnectionMade
+ self.factory.protocolConnectionMade = None
+ d.callback(self)
+
+ def dataReceived(self, data):
+ self.data += data
+
+ def connectionLost(self, reason):
+ self.closed = 1
+ self.closedReason = reason
+ if self.closedDeferred is not None:
+ d, self.closedDeferred = self.closedDeferred, None
+ d.callback(None)
+
+ at inlineCallbacks
+def getURL(url, method="GET", redirect=0):
+
+ agent = Agent(reactor)
+ headers = http_headers.Headers({})
+
+ try:
+ response = (yield agent.request(method, url, headers, None))
+ except Exception, e:
+ log.err(str(e))
+ response = None
+ else:
+ if response.code in (MOVED_PERMANENTLY, FOUND, TEMPORARY_REDIRECT,):
+ if redirect > 3:
+ log.err("Too many redirects")
+ else:
+ location = response.headers.getRawHeaders("location")
+ if location:
+ newresponse = (yield getURL(location[0], method=method, redirect=redirect+1))
+ if response.code == MOVED_PERMANENTLY:
+ scheme, netloc, url, _ignore_params, _ignore_query, _ignore_fragment = urlparse(location[0])
+ newresponse.location = urlunparse((scheme, netloc, url, None, None, None,))
+ returnValue(newresponse)
+ else:
+ log.err("Redirect without a Location header")
+
+ if response is not None and response.code / 100 == 2:
+ protocol = AccumulatingProtocol()
+ response.deliverBody(protocol)
+ whenFinished = protocol.closedDeferred = Deferred()
+ yield whenFinished
+ response.data = protocol.data
+ else:
+ log.error("Failed getURL: %s" % (url,))
+
+ returnValue(response)
+
Modified: CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/timezonestdservice.py
===================================================================
--- CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/timezonestdservice.py 2011-05-23 21:42:12 UTC (rev 7517)
+++ CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/timezonestdservice.py 2011-05-24 00:38:32 UTC (rev 7518)
@@ -37,22 +37,29 @@
from twext.web2.http_headers import MimeType
from twext.web2.stream import MemoryStream
-from twisted.internet.defer import succeed
+from twisted.internet import reactor
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue
+from twisted.python.log import addObserver, removeObserver
from twistedcaldav import timezonexml, xmlutil
+from twistedcaldav.client.geturl import getURL
+from twistedcaldav.config import config
from twistedcaldav.customxml import calendarserver_namespace
from twistedcaldav.extensions import DAVResource,\
DAVResourceWithoutChildrenMixin
from twistedcaldav.ical import tzexpandlocal
from twistedcaldav.resource import ReadOnlyNoCopyResourceMixIn
from twistedcaldav.timezones import TimezoneException, TimezoneCache, readVTZ
-from twistedcaldav.xmlutil import addSubElement
+from twistedcaldav.xmlutil import addSubElement, readXMLString
from pycalendar.calendar import PyCalendar
from pycalendar.datetime import PyCalendarDateTime
+from pycalendar.exceptions import PyCalendarInvalidData
+from sys import stdout, stderr
import getopt
import hashlib
+import itertools
import os
import sys
@@ -441,7 +448,7 @@
xmlutil.addSubElement(node, "dtstamp", self.dtstamp)
xmlutil.addSubElement(node, "md5", self.md5)
-class PrimaryTimezoneDatabase(object):
+class CommonTimezoneDatabase(object):
"""
Maintains the database of timezones read from an XML file.
"""
@@ -452,6 +459,52 @@
self.dtstamp = None
self.timezones = {}
+ def readDatabase(self):
+ """
+ Read in XML data.
+ """
+ _ignore, root = xmlutil.readXML(self.xmlfile, "timezones")
+ self.dtstamp = root.findtext("dtstamp")
+ for child in root.getchildren():
+ if child.tag == "timezone":
+ tz = TimezoneInfo.readXML(child)
+ if tz:
+ self.timezones[tz.tzid] = tz
+
+ def getTimezones(self, tzids):
+ """
+ Generate a PyCalendar containing the requested timezones.
+ """
+ calendar = PyCalendar()
+ for tzid in sorted(tzids):
+ # We will just use our existing TimezoneCache here
+ try:
+ vtz = readVTZ(tzid)
+ calendar.addComponent(vtz.getComponents()[0].duplicate())
+ except TimezoneException:
+ pass
+
+ return calendar
+
+ def _dumpTZs(self):
+
+ _ignore, root = xmlutil.newElementTreeWithRoot("timezones")
+ addSubElement(root, "dtstamp", self.dtstamp)
+ for _ignore,v in sorted(self.timezones.items(), key=lambda x:x[0]):
+ v.generateXML(root)
+ xmlutil.writeXML(self.xmlfile, root)
+
+class PrimaryTimezoneDatabase(CommonTimezoneDatabase):
+ """
+ Maintains the database of timezones read from an XML file.
+ """
+
+ def __init__(self, basepath, xmlfile):
+ self.basepath = basepath
+ self.xmlfile = xmlfile
+ self.dtstamp = None
+ self.timezones = {}
+
def createNewDatabase(self):
"""
Create a new DB xml file from scratch by scanning zoneinfo.
@@ -484,14 +537,6 @@
self.changed.add(tzid)
self.timezones[tzid] = TimezoneInfo(tzid, self.dtstamp, md5)
- def _dumpTZs(self):
-
- _ignore, root = xmlutil.newElementTreeWithRoot("timezones")
- addSubElement(root, "dtstamp", self.dtstamp)
- for _ignore,v in sorted(self.timezones.items(), key=lambda x:x[0]):
- v.generateXML(root)
- xmlutil.writeXML(self.xmlfile, root)
-
def updateDatabase(self):
"""
Update existing DB info by comparing md5's.
@@ -502,34 +547,262 @@
self._scanTZs("", checkIfChanged=True)
if self.changeCount:
self._dumpTZs()
+
+class SecondaryTimezoneDatabase(CommonTimezoneDatabase):
+ """
+ Caches a database of timezones from another timezone service.
+ """
- def readDatabase(self):
+ def __init__(self, basepath, xmlfile, uri):
+ self.basepath = basepath
+ self.xmlfile = xmlfile
+ self.uri = uri
+ self.discovered = False
+ self.dtstamp = None
+ self.timezones = {}
+ self._url = None
+
+ if not os.path.exists(self.basepath):
+ os.makedirs(self.basepath)
+
+ @inlineCallbacks
+ def syncWithServer(self):
"""
- Read in XML data.
+ Sync local data with that from the server we are replicating.
"""
- _ignore, root = xmlutil.readXML(self.xmlfile, "timezones")
- self.dtstamp = root.findtext("dtstamp")
- for child in root.getchildren():
- if child.tag == "timezone":
- tz = TimezoneInfo.readXML(child)
- if tz:
- self.timezones[tz.tzid] = tz
+
+ result = (yield self._getTimezoneListFromServer())
+ if result is None:
+ # Nothing changed since last sync
+ returnValue(None)
+ newdtstamp, newtimezones = result
+
+ # Compare timezone infos
+
+ # New ones on the server
+ newtzids = set(newtimezones.keys()) - set(self.timezones.keys())
+
+ # Check for changes
+ changedtzids = set()
+ for tzid in set(newtimezones.keys()) & set(self.timezones.keys()):
+ if self.timezones[tzid].dtstamp < newtimezones[tzid].dtstamp:
+ changedtzids.add(tzid)
+
+ # Now apply changes
+ for tzid in itertools.chain(newtzids, changedtzids):
+ yield self._getTimezoneFromServer(newtimezones[tzid])
+
+ self.dtstamp = newdtstamp
+ self._dumpTZs()
+
+ returnValue((len(newtzids), len(changedtzids),))
+
+ @inlineCallbacks
+ def _discoverServer(self):
+ """
+ Make sure we know the timezone service path
+ """
+
+ if self.uri is None:
+ if config.TimezoneService.SecondaryService.Host:
+ self.uri = "https://%s/.well-known/timezone" % (self.config.TimezoneService.SecondaryService.Host,)
+ elif config.TimezoneService.SecondaryService.URI:
+ self.uri = config.TimezoneService.SecondaryService.URI
+ elif not self.uri.startswith("https:") and not self.uri.startswith("http:"):
+ self.uri = "https://%s/.well-known/timezone" % (self.uri,)
+
+ testURI = "%s?action=capabilities" % (self.uri,)
+ response = (yield getURL(testURI))
+ if response is None or response.code / 100 != 2:
+ self.discovered = False
+ returnValue(False)
+
+ # Cache the redirect target
+ if hasattr(response, "location"):
+ self.uri = response.location
- def getTimezones(self, tzids):
+ # TODO: Ignoring the data from capabilities for now
+
+ self.discovered = True
+ returnValue(True)
+
+ @inlineCallbacks
+ def _getTimezoneListFromServer(self):
"""
- Generate a PyCalendar containing the requested timezones.
+ Retrieve the timezone list from the specified server
"""
- calendar = PyCalendar()
- for tzid in sorted(tzids):
- # We will just use our existing TimezoneCache here
- try:
- vtz = readVTZ(tzid)
- calendar.addComponent(vtz.getComponents()[0].duplicate())
- except TimezoneException:
- pass
+
+ # Make sure we have the server
+ if not self.discovered:
+ result = (yield self._discoverServer())
+ if not result:
+ returnValue(None)
+
+ # List all from the server
+ url = "%s?action=list" % (self.uri,)
+ if self.dtstamp:
+ url = "%s&changesince=%s" % (url, self.dtstamp,)
+ response = (yield getURL(url))
+ if response is None or response.code / 100 != 2:
+ returnValue(None)
+
+ ct = response.headers.getRawHeaders("content-type", ("bogus/type",))[0]
+ ct = ct.split(";", 1)
+ ct = ct[0]
+ if ct not in ("application/xml", "text/xml",):
+ returnValue(None)
+
+ etroot, _ignore = readXMLString(response.data, timezonexml.TimezoneList.sname())
+ dtstamp = etroot.findtext(timezonexml.Dtstamp.sname())
+ timezones = {}
+ for summary in etroot.findall(timezonexml.Summary.sname()):
+ tzid = summary.findtext(timezonexml.Tzid.sname())
+ lastmod = summary.findtext(timezonexml.LastModified.sname())
+ timezones[tzid] = TimezoneInfo(tzid, lastmod, None)
+
+ returnValue((dtstamp, timezones,))
- return calendar
+ @inlineCallbacks
+ def _getTimezoneFromServer(self, tzinfo):
+ # List all from the server
+ response = (yield getURL("%s?action=get&tzid=%s" % (self.uri, tzinfo.tzid,)))
+ if response is None or response.code / 100 != 2:
+ returnValue(None)
+
+ ct = response.headers.getRawHeaders("content-type", ("bogus/type",))[0]
+ ct = ct.split(";", 1)
+ ct = ct[0]
+ if ct not in ("text/calendar",):
+ log.error("Invalid content-type '%s' for tzid : %s" % (ct, tzinfo.tzid,))
+ returnValue(None)
+
+ ical = response.data
+ try:
+ calendar = PyCalendar.parseText(ical)
+ except PyCalendarInvalidData:
+ log.error("Invalid calendar data for tzid: %s" % (tzinfo.tzid,))
+ returnValue(None)
+ ical = calendar.getText()
+ tzinfo.md5 = hashlib.md5(ical).hexdigest()
+
+ try:
+ tzpath = os.path.join(self.basepath, tzinfo.tzid) + ".ics"
+ if not os.path.exists(os.path.dirname(tzpath)):
+ os.makedirs(os.path.dirname(tzpath))
+ f = open(tzpath, "w")
+ f.write(ical)
+ f.close()
+ except IOError, e:
+ log.error("Unable to write calendar file for %s: %s" % (tzinfo.tzid, str(e),))
+ else:
+ self.timezones[tzinfo.tzid] = tzinfo
+
+ def _removeTimezone(self, tzid):
+ tzpath = os.path.join(self.basepath, tzid) + ".ics"
+ try:
+ os.remove(tzpath)
+ del self.timezones[tzid]
+ except IOError, e:
+ log.error("Unable to write calendar file for %s: %s" % (tzid, str(e),))
+
+def _doPrimaryActions(action, tzpath, xmlfile, changed):
+
+ tzdb = PrimaryTimezoneDatabase(tzpath, xmlfile)
+ if action == "create":
+ print "Creating new XML file at: %s" % (xmlfile, )
+ tzdb.createNewDatabase()
+ print "Current total: %d" % (len(tzdb.timezones), )
+
+ elif action == "update":
+ print "Updating XML file at: %s" % (xmlfile, )
+ tzdb.readDatabase()
+ tzdb.updateDatabase()
+ print "Current total: %d" % (len(tzdb.timezones), )
+ print "Total Changed: %d" % (tzdb.changeCount, )
+ if tzdb.changeCount:
+ print "Changed:"
+ for k in sorted(tzdb.changed):
+ print " %s" % (k, )
+
+ elif action == "list":
+ print "Listing XML file at: %s" % (xmlfile, )
+ tzdb.readDatabase()
+ print "Current timestamp: %s" % (tzdb.dtstamp, )
+ print "Timezones:"
+ for k in sorted(tzdb.timezones.keys()):
+ print " %s" % (k, )
+
+ elif action == "changed":
+ print "Changes from XML file at: %s" % (xmlfile, )
+ tzdb.readDatabase()
+ print "Check timestamp: %s" % (changed, )
+ print "Current timestamp: %s" % (tzdb.dtstamp, )
+ results = [k for k, v in tzdb.timezones.items() if v.dtstamp > changed]
+ print "Total Changed: %d" % (len(results), )
+ if results:
+ print "Changed:"
+ for k in sorted(results):
+ print " %s" % (k, )
+ else:
+ usage("Invalid action: %s" % (action, ))
+
+class StandardIOObserver (object):
+ """
+ Log observer that writes to standard I/O.
+ """
+ def emit(self, eventDict):
+ text = None
+
+ if eventDict["isError"]:
+ output = stderr
+ if "failure" in eventDict:
+ text = eventDict["failure"].getTraceback()
+ else:
+ output = stdout
+
+ if not text:
+ text = " ".join([str(m) for m in eventDict["message"]]) + "\n"
+
+ output.write(text)
+ output.flush()
+
+ def start(self):
+ addObserver(self.emit)
+
+ def stop(self):
+ removeObserver(self.emit)
+
+ at inlineCallbacks
+def _runInReactor(tzdb):
+
+ try:
+ new, changed = yield tzdb.syncWithServer()
+ print "New: %d" % (new, )
+ print "Changed: %d" % (changed, )
+ print "Current total: %d" % (len(tzdb.timezones), )
+ except Exception, e:
+ print "Could not sync with server: %s" % (str(e),)
+ finally:
+ reactor.stop()
+
+def _doSecondaryActions(action, tzpath, xmlfile, url):
+
+ tzdb = SecondaryTimezoneDatabase(tzpath, xmlfile, url)
+ try:
+ tzdb.readDatabase()
+ except:
+ pass
+ if action == "cache":
+ print "Caching from secondary server: %s" % (url, )
+
+ observer = StandardIOObserver()
+ observer.start()
+ reactor.callLater(0, _runInReactor, tzdb)
+ reactor.run()
+ else:
+ usage("Invalid action: %s" % (action, ))
+
def usage(error_msg=None):
if error_msg:
print error_msg
@@ -541,10 +814,16 @@
-v Be verbose
-f XML file path
-z zoneinfo file path
+
+ # Primary service
--create create new XML file
--update update XML file
--list list timezones in XML file
--changed changed since timestamp
+
+ # Secondary service
+ --url URL or domain of service
+ --cache Cache data from service
Description:
This utility will create, update or list an XML timezone database
@@ -559,10 +838,13 @@
if __name__ == '__main__':
+ primary = False
+ secondary = False
action = None
tzpath = None
xmlfile = None
changed = None
+ url = None
# Get options
options, args = getopt.getopt(
@@ -572,7 +854,9 @@
"create",
"update",
"list",
- "changed="
+ "changed=",
+ "url=",
+ "cache",
]
)
@@ -587,18 +871,29 @@
tzpath = value
elif option == "--create":
action = "create"
+ primary = True
elif option == "--update":
action = "update"
+ primary = True
elif option == "--list":
action = "list"
+ primary = True
elif option == "--changed":
action = "changed"
+ primary = True
changed = value
+ elif option == "--url":
+ url = value
+ secondary = True
+ elif option == "--cache":
+ action = "cache"
+ secondary = True
else:
usage("Unrecognized option: %s" % (option,))
if action is None:
action = "list"
+ primary = True
if tzpath is None:
try:
import pkg_resources
@@ -608,45 +903,15 @@
tzpath = pkg_resources.resource_filename("twistedcaldav", "zoneinfo") #@UndefinedVariable
xmlfile = os.path.expanduser("~/tz.xml")
- if not os.path.isdir(tzpath):
+ if primary and not os.path.isdir(tzpath):
usage("Invalid zoneinfo path: %s" % (tzpath,))
- if not os.path.isfile(xmlfile) and action != "create":
+ if primary and not os.path.isfile(xmlfile) and action != "create":
usage("Invalid XML file path: %s" % (xmlfile,))
- tzdb = PrimaryTimezoneDatabase(tzpath, xmlfile)
- if action == "create":
- print "Creating new XML file at: %s" % (xmlfile,)
- tzdb.createNewDatabase()
- print "Current total: %d" % (len(tzdb.timezones),)
- elif action == "update":
- print "Updating XML file at: %s" % (xmlfile,)
- tzdb.readDatabase()
- tzdb.updateDatabase()
- print "Current total: %d" % (len(tzdb.timezones),)
- print "Total Changed: %d" % (tzdb.changeCount,)
- if tzdb.changeCount:
- print "Changed:"
- for k in sorted(tzdb.changed):
- print " %s" % (k,)
- elif action == "list":
- print "Listing XML file at: %s" % (xmlfile,)
- tzdb.readDatabase()
- print "Current timestamp: %s" % (tzdb.dtstamp,)
- print "Timezones:"
- for k in sorted(tzdb.timezones.keys()):
- print " %s" % (k,)
- elif action == "changed":
- print "Changes from XML file at: %s" % (xmlfile,)
- tzdb.readDatabase()
- print "Check timestamp: %s" % (changed,)
- print "Current timestamp: %s" % (tzdb.dtstamp,)
- results = [k for k, v in tzdb.timezones.items() if v.dtstamp > changed]
- print "Total Changed: %d" % (len(results),)
- if results:
- print "Changed:"
- for k in sorted(results):
- print " %s" % (k,)
+ if primary and secondary:
+ usage("Cannot use primary and secondary options together")
+
+ if primary:
+ _doPrimaryActions(action, tzpath, xmlfile, changed)
else:
- usage("Invalid action: %s" % (action,))
-
-
\ No newline at end of file
+ _doSecondaryActions(action, tzpath, xmlfile, url)
Modified: CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/xmlutil.py
===================================================================
--- CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/xmlutil.py 2011-05-23 21:42:12 UTC (rev 7517)
+++ CalendarServer/branches/users/cdaboo/timezones/twistedcaldav/xmlutil.py 2011-05-24 00:38:32 UTC (rev 7518)
@@ -16,7 +16,9 @@
from __future__ import with_statement
+import cStringIO as StringIO
import xml.etree.ElementTree as XML
+
try:
from xml.etree.ElementTree import ParseError as XMLParseError
except ImportError:
@@ -24,6 +26,10 @@
# Utilities for working with ElementTree
+def readXMLString(xmldata, expectedRootTag=None):
+ io = StringIO.StringIO(xmldata)
+ return readXML(io, expectedRootTag)
+
def readXML(xmlfile, expectedRootTag=None):
"""
Read in XML data from a file and parse into ElementTree. Optionally verify
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110523/0a1f39ec/attachment-0001.html>
More information about the calendarserver-changes
mailing list