[CalendarServer-changes] [6156] CalendarServer/trunk/contrib/performance
source_changes at macosforge.org
source_changes at macosforge.org
Fri Aug 20 10:44:19 PDT 2010
Revision: 6156
http://trac.macosforge.org/projects/calendarserver/changeset/6156
Author: exarkun at twistedmatrix.com
Date: 2010-08-20 10:44:15 -0700 (Fri, 20 Aug 2010)
Log Message:
-----------
Switch to incremental dtrace output parsing and use t.w.client instead of CalDAVClientLibrary to avoid the httplib EINTR problems
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/benchlib.py
CalendarServer/trunk/contrib/performance/benchmark.py
CalendarServer/trunk/contrib/performance/event.py
CalendarServer/trunk/contrib/performance/sqlwatch.py
CalendarServer/trunk/contrib/performance/vfreebusy.py
Modified: CalendarServer/trunk/contrib/performance/benchlib.py
===================================================================
--- CalendarServer/trunk/contrib/performance/benchlib.py 2010-08-20 17:16:53 UTC (rev 6155)
+++ CalendarServer/trunk/contrib/performance/benchlib.py 2010-08-20 17:44:15 UTC (rev 6156)
@@ -2,25 +2,62 @@
from time import time
from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.web.http_headers import Headers
-from client.account import CalDAVAccount
from protocol.url import URL
from stats import Duration
-from httpclient import readBody
+from httpclient import StringProducer, readBody
-def initialize(host, port, user, password, root, principal, calendar):
+class CalDAVAccount(object):
+ def __init__(self, agent, netloc, user, password, root, principal):
+ self.agent = agent
+ self.netloc = netloc
+ self.user = user
+ self.password = password
+ self.root = root
+ self.principal = principal
+
+ def deleteResource(self, url):
+ return self.agent.request('DELETE', 'http://%s%s' % (self.netloc, url.toString()))
+
+
+ def makeCalendar(self, url):
+ return self.agent.request('MKCALENDAR', 'http://%s%s' % (self.netloc, url.toString()))
+
+
+ def writeData(self, url, data, contentType):
+ return self.agent.request(
+ 'PUT',
+ 'http://%s%s' % (self.netloc, url.toString()),
+ Headers({'content-type': [contentType]}),
+ StringProducer(data))
+
+
+
+ at inlineCallbacks
+def _serial(fs):
+ for (f, args) in fs:
+ yield f(*args)
+ returnValue(None)
+
+
+
+def initialize(agent, host, port, user, password, root, principal, calendar):
"""
If the specified calendar exists, delete it. Then re-create it empty.
"""
account = CalDAVAccount(
+ agent,
"%s:%d" % (host, port),
- user=user, pswd=password,
+ user=user, password=password,
root=root, principal=principal)
- cal = "/calendars/users/%s/%s/" % (user, calendar)
- account.session.deleteResource(URL(cal))
- account.session.makeCalendar(URL(cal))
- return account
+ cal = URL("/calendars/users/%s/%s/" % (user, calendar))
+ d = _serial([
+ (account.deleteResource, (cal,)),
+ (account.makeCalendar, (cal,))])
+ d.addCallback(lambda ignored: account)
+ return d
@inlineCallbacks
Modified: CalendarServer/trunk/contrib/performance/benchmark.py
===================================================================
--- CalendarServer/trunk/contrib/performance/benchmark.py 2010-08-20 17:16:53 UTC (rev 6155)
+++ CalendarServer/trunk/contrib/performance/benchmark.py 2010-08-20 17:44:15 UTC (rev 6156)
@@ -5,16 +5,15 @@
from pickle import dump
from datetime import datetime
-from StringIO import StringIO
from twisted.python.reflect import namedAny
from twisted.internet.protocol import ProcessProtocol
+from twisted.protocols.basic import LineReceiver
from twisted.internet.defer import (
Deferred, inlineCallbacks, gatherResults)
from twisted.internet import reactor
from stats import SQLDuration, Bytes
-import vfreebusy
class DTraceBug(Exception):
@@ -26,13 +25,13 @@
class IOMeasureConsumer(ProcessProtocol):
- def __init__(self, started, done):
+ def __init__(self, started, done, parser):
self.started = started
self.done = done
+ self.parser = parser
def connectionMade(self):
- self.out = StringIO()
self._out = ''
self._err = ''
@@ -47,11 +46,11 @@
def outReceived(self, bytes):
if self.started is None:
- self.out.write(bytes)
+ self.parser.dataReceived(bytes)
else:
self._out += bytes
if self._out.startswith('READY\n'):
- self.out.write(self._out[len('READY\n'):])
+ self.parser.dataReceived(self._out[len('READY\n'):])
del self._out
started = self.started
self.started = None
@@ -60,7 +59,7 @@
def processEnded(self, reason):
if self.started is None:
- self.done.callback(self.out.getvalue())
+ self.done.callback(None)
else:
self.started.errback(RuntimeError("Exited too soon"))
@@ -76,39 +75,22 @@
return pids
-class DTraceCollector(object):
- def __init__(self, script, pids):
- self._dScript = script
- self.pids = pids
- self._read = []
- self._write = []
- self._execute = []
- self._iternext = []
+class _DTraceParser(LineReceiver):
+ delimiter = '\n\1'
+ sql = None
+ start = None
- def stats(self):
- return {
- Bytes('read'): self._read,
- Bytes('write'): self._write,
- SQLDuration('execute'): self._execute,
- SQLDuration('iternext'): self._iternext,
- }
+ def __init__(self, collector):
+ self.collector = collector
- def _parse(self, dtrace):
- file('dtrace.log', 'a').write(dtrace)
-
- self.sql = self.start = None
- for L in dtrace.split('\n\1'):
-
- # dtrace puts some extra newlines in the output sometimes. Get rid of them.
- L = L.strip()
- if not L:
- continue
-
- op, rest = L.split(None, 1)
+ def lineReceived(self, dtrace):
+ # dtrace puts some extra newlines in the output sometimes. Get rid of them.
+ dtrace = dtrace.strip()
+ if dtrace:
+ op, rest = dtrace.split(None, 1)
getattr(self, '_op_' + op)(op, rest)
- self.sql = self.start = None
def _op_EXECUTE(self, cmd, rest):
@@ -129,9 +111,9 @@
print 'Completely bogus EXECUTE', self.start, when
else:
if cmd == 'EXECUTE':
- accum = self._execute
+ accum = self.collector._execute
elif cmd == 'ITERNEXT':
- accum = self._iternext
+ accum = self.collector._iternext
accum.append((self.sql, diff))
self.start = None
@@ -139,13 +121,33 @@
_op_ITERNEXT = _op_EXECUTE
def _op_B_READ(self, cmd, rest):
- self._read.append(int(rest))
+ self.collector._read.append(int(rest))
def _op_B_WRITE(self, cmd, rest):
- self._write.append(int(rest))
+ self.collector._write.append(int(rest))
+
+class DTraceCollector(object):
+ def __init__(self, script, pids):
+ self._dScript = script
+ self.pids = pids
+ self._read = []
+ self._write = []
+ self._execute = []
+ self._iternext = []
+
+
+ def stats(self):
+ return {
+ Bytes('read'): self._read,
+ Bytes('write'): self._write,
+ SQLDuration('execute'): self._execute,
+ SQLDuration('iternext'): self._iternext,
+ }
+
+
def start(self):
ready = []
self.finished = []
@@ -161,7 +163,7 @@
started = Deferred()
stopped = Deferred()
process = reactor.spawnProcess(
- IOMeasureConsumer(started, stopped),
+ IOMeasureConsumer(started, stopped, _DTraceParser(self)),
"/usr/sbin/dtrace",
["/usr/sbin/dtrace",
# process preprocessor macros
@@ -185,7 +187,6 @@
# processes.
self.dtraces[pid] = process
stopped.addCallback(self._cleanup, pid)
- stopped.addCallback(self._parse)
return passthrough
started.addCallbacks(ready, eintr)
return started, stopped
Modified: CalendarServer/trunk/contrib/performance/event.py
===================================================================
--- CalendarServer/trunk/contrib/performance/event.py 2010-08-20 17:16:53 UTC (rev 6155)
+++ CalendarServer/trunk/contrib/performance/event.py 2010-08-20 17:44:15 UTC (rev 6156)
@@ -9,6 +9,7 @@
from uuid import uuid4
from datetime import datetime, timedelta
+from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
@@ -88,6 +89,7 @@
}
+ at inlineCallbacks
def measure(dtrace, attendeeCount, samples):
user = password = "user01"
host = "localhost"
@@ -96,18 +98,17 @@
principal = "/"
calendar = "event-creation-benchmark"
- # First set things up
- initialize(host, port, user, password, root, principal, calendar)
-
- # CalDAVClientLibrary can't seem to POST things. Use Twisted instead.
authinfo = HTTPDigestAuthHandler()
authinfo.add_password(
realm="Test Realm",
uri="http://%s:%d/" % (host, port),
user=user,
passwd=password)
+ agent = AuthHandlerAgent(Agent(reactor), authinfo)
- agent = AuthHandlerAgent(Agent(reactor), authinfo)
+ # First set things up
+ yield initialize(agent, host, port, user, password, root, principal, calendar)
+
method = 'PUT'
uri = 'http://%s:%d/calendars/__uids__/%s/%s/foo-%%d.ics' % (
host, port, user, calendar)
@@ -117,8 +118,10 @@
events = ((i, makeEvent(i, attendeeCount)) for i in count(2))
# Sample it a bunch of times
- return sample(
+ samples = yield sample(
dtrace, samples,
agent, ((method, uri % (i,), headers, StringProducer(body))
for (i, body)
in events).next)
+ returnValue(samples)
+
Modified: CalendarServer/trunk/contrib/performance/sqlwatch.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlwatch.py 2010-08-20 17:16:53 UTC (rev 6155)
+++ CalendarServer/trunk/contrib/performance/sqlwatch.py 2010-08-20 17:44:15 UTC (rev 6156)
@@ -1,6 +1,5 @@
-import sys, os, signal, time
-from pprint import pprint
+import sys, signal, time
from twisted.python.log import err
from twisted.python.failure import Failure
Modified: CalendarServer/trunk/contrib/performance/vfreebusy.py
===================================================================
--- CalendarServer/trunk/contrib/performance/vfreebusy.py 2010-08-20 17:16:53 UTC (rev 6155)
+++ CalendarServer/trunk/contrib/performance/vfreebusy.py 2010-08-20 17:44:15 UTC (rev 6156)
@@ -8,8 +8,7 @@
from protocol.url import URL
-from twisted.internet.defer import (
- inlineCallbacks)
+from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
@@ -97,6 +96,7 @@
return [makeEvent(i) for i in range(n)]
+ at inlineCallbacks
def measure(dtrace, events, samples):
user = password = "user01"
host = "localhost"
@@ -105,28 +105,28 @@
principal = "/"
calendar = "vfreebusy-benchmark"
- # First set things up
- account = initialize(host, port, user, password, root, principal, calendar)
-
- base = "/calendars/users/%s/%s/foo-%%d.ics" % (user, calendar)
- for i, cal in enumerate(makeEvents(events)):
- account.session.writeData(
- URL(base % (i,)), cal, "text/calendar")
-
- # CalDAVClientLibrary can't seem to POST things.
authinfo = HTTPDigestAuthHandler()
authinfo.add_password(
realm="Test Realm",
uri="http://%s:%d/" % (host, port),
user=user,
passwd=password)
+ agent = AuthHandlerAgent(Agent(reactor), authinfo)
- agent = AuthHandlerAgent(Agent(reactor), authinfo)
+ # First set things up
+ account = yield initialize(agent, host, port, user, password, root, principal, calendar)
+
+ base = "/calendars/users/%s/%s/foo-%%d.ics" % (user, calendar)
+ for i, cal in enumerate(makeEvents(events)):
+ yield account.writeData(URL(base % (i,)), cal, "text/calendar")
+
method = 'POST'
- uri = 'http://localhost:8008/calendars/__uids__/user01/outbox/'
+ uri = 'http://localhost:8008/calendars/__uids__/%s/outbox/' % (user,)
headers = Headers({"content-type": ["text/calendar"]})
body = StringProducer(vfreebusy)
- return sample(
+ samples = yield sample(
dtrace, samples,
agent, lambda: (method, uri, headers, body))
+ returnValue(samples)
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100820/9c792abb/attachment-0001.html>
More information about the calendarserver-changes
mailing list