[CalendarServer-changes] [1948] CalendarClientSimulator/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Oct 9 10:51:22 PDT 2007
Revision: 1948
http://trac.macosforge.org/projects/calendarserver/changeset/1948
Author: cdaboo at apple.com
Date: 2007-10-09 10:51:21 -0700 (Tue, 09 Oct 2007)
Log Message:
-----------
Multi-threaded client simulator.
Modified Paths:
--------------
CalendarClientSimulator/trunk/ReadMe.txt
CalendarClientSimulator/trunk/src/calendarclient.py
CalendarClientSimulator/trunk/src/multisim.py
Added Paths:
-----------
CalendarClientSimulator/trunk/src/clientscheduler.py
CalendarClientSimulator/trunk/src/taskscheduler.py
CalendarClientSimulator/trunk/src/threadedsim.py
Modified: CalendarClientSimulator/trunk/ReadMe.txt
===================================================================
--- CalendarClientSimulator/trunk/ReadMe.txt 2007-10-06 03:10:01 UTC (rev 1947)
+++ CalendarClientSimulator/trunk/ReadMe.txt 2007-10-09 17:51:21 UTC (rev 1948)
@@ -98,3 +98,30 @@
option). Each simulation is a seperate process running the simulate tool, with the options set for each user
number. The tool starts all the simulations and then waits for a key press, after which it kills all the simulation
processes.
+
+Usage: threadedsim [options]
+Options:
+ --pool number of threads processing clients [10]
+ --number number of users to simulate [10]
+ --server URL for server (e.g. https://caldav.example.com:8443) [Required]
+ --user user id for user to login as [user%02d]
+ --password password for user [user%02d]
+ --interval polling interval in seconds [15 mins]
+ --eventsperday number of events per day to create [10]
+ --invitesperday number of invites per day to send [5]
+ --cache path to .plist file to cache data [../data/user%02d.plist]
+ --clear-cache clear the cache when starting up [Optional]
+ --no-throttle do not throttle the task queue when scheduler is too busy [Optional]
+ --verbose print out activity log
+ --logging log activity
+
+ -h, --help print this help and exit
+
+This tool generates multiple client simulations for a range of users (1 through the number specified with the --number
+option) and runs a single multi-threaded process to handle all client activity. The tool starts all the simulations and
+then waits for a key press, after which it stops all the simulation threads and then exists. You will need to tune the
+thread pool size based on the number of clients and activity level and server load. The tool will print out status
+when it starts to get overloaded and a warning when all threads are busy. If throttling is enabled the tool will stop
+client requests being queued when the queue size is 2x the thread pool size. It will re-enable queing once the number
+of busy threads is 1/2 the thread pool size.
+
Modified: CalendarClientSimulator/trunk/src/calendarclient.py
===================================================================
--- CalendarClientSimulator/trunk/src/calendarclient.py 2007-10-06 03:10:01 UTC (rev 1947)
+++ CalendarClientSimulator/trunk/src/calendarclient.py 2007-10-09 17:51:21 UTC (rev 1948)
@@ -255,7 +255,33 @@
self.doInvite()
time.sleep(CalendarClient.sleep)
- def doPoll(self):
+ def start(self):
+
+ self.log("Starting CalendarClient simulation for user %s" % (self.user,))
+ self.doPoll()
+
+ self.start_poll = time.time() - randint(0, self.interval - 1)
+ self.event_interval = 24 * 60 * 60 / self.eventsperday
+ self.invite_interval = 24 * 60 * 60 / self.invitesperday
+ self.start_events = time.time() - randint(0, self.event_interval - 1)
+ self.start_invites = time.time() - randint(0, self.invite_interval - 1)
+
+ def pending(self):
+
+ run_tasks = []
+ if time.time() >= self.start_poll + self.interval:
+ self.start_poll = time.time()
+ run_tasks.append((self.doPoll, (), {},))
+ if time.time() >= self.start_events + self.event_interval:
+ self.start_events = time.time()
+ run_tasks.append((self.doCreateEvent, (), {},))
+ if time.time() >= self.start_invites + self.invite_interval:
+ self.start_invites = time.time()
+ run_tasks.append((self.doInvite, (), {},))
+
+ return run_tasks
+
+ def doPoll(self, *args, **kwargs):
self.log("Polling: %s" % (self.user,))
try:
status, _ignore_headers, data = self.doRequest(self.home, "PROPFIND", {"Content-Type": "application/xml", "Depth":"1"}, PROPFIND_ctag)
@@ -474,7 +500,7 @@
# attendees.append("/principals/users/user01/")
return attendees
- def doInvite(self):
+ def doInvite(self, *args, **kwargs):
# Generate data for this user
organizer = "/principals/users/%s/" % (self.user,)
Added: CalendarClientSimulator/trunk/src/clientscheduler.py
===================================================================
--- CalendarClientSimulator/trunk/src/clientscheduler.py (rev 0)
+++ CalendarClientSimulator/trunk/src/clientscheduler.py 2007-10-09 17:51:21 UTC (rev 1948)
@@ -0,0 +1,70 @@
+##
+# Copyright (c) 2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DRI: Cyrus Daboo, cdaboo at apple.com
+##
+
+from taskscheduler import taskscheduler
+from threading import Thread
+import time
+
+# Manage scheduling of client tasks.
+
+class ClientScheduler(object):
+
+ sleep = 0.1
+
+ def __init__(self, num_threads=10, throttle=True):
+ self.running = False
+ self.clients = []
+ self.tasks = taskscheduler(num_threads, throttle)
+ self.thread = None
+
+ def add(self, client):
+ self.clients.append(client)
+ client.start()
+
+ def run(self):
+
+ print "Starting client scheduler"
+ self.running = True
+
+ def do_work():
+ while(self.running):
+ try:
+ # Loop over each client and ask it whether it has a pending task
+ for client in self.clients:
+ pending = client.pending()
+ if pending:
+ for callback, args, kwargs in pending:
+ self.tasks.enqueue(callback, args, kwargs)
+ time.sleep(ClientScheduler.sleep)
+ except:
+ pass
+
+ # Generate a pool of threads
+ self.thread = Thread(target=do_work)
+ self.thread.start()
+
+ print "Started client scheduler"
+
+ self.tasks.run()
+
+ def stop(self):
+ print "Stopping client scheduler"
+ self.running = False
+ self.thread.join()
+ self.tasks.stop()
+ print "Stopped client scheduler"
Modified: CalendarClientSimulator/trunk/src/multisim.py
===================================================================
--- CalendarClientSimulator/trunk/src/multisim.py 2007-10-06 03:10:01 UTC (rev 1947)
+++ CalendarClientSimulator/trunk/src/multisim.py 2007-10-09 17:51:21 UTC (rev 1948)
@@ -29,10 +29,10 @@
# Simulate a whole bunch of users
def usage():
- print """Usage: simulate [options]
+ print """Usage: multisim [options]
Options:
--number number of users to simulate [10]
- --startat user number to start at
+ --startat user number to start at [1]
--server URL for server (e.g. https://caldav.example.com:8443) [Required]
--user user id for user to login as [user%02d]
--password password for user [user%02d]
@@ -49,7 +49,7 @@
if __name__ == '__main__':
- count = 5
+ count = 10
startat = 1
server = None
user = "user%02d"
Added: CalendarClientSimulator/trunk/src/taskscheduler.py
===================================================================
--- CalendarClientSimulator/trunk/src/taskscheduler.py (rev 0)
+++ CalendarClientSimulator/trunk/src/taskscheduler.py 2007-10-09 17:51:21 UTC (rev 1948)
@@ -0,0 +1,109 @@
+##
+# Copyright (c) 2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DRI: Cyrus Daboo, cdaboo at apple.com
+##
+from threading import Thread
+import Queue
+
+# Schedule calendar client tasks and execute via a thread pool.
+
+class taskscheduler(object):
+
+ def __init__(self, num_threads=10, throttle=True):
+ self.queue = Queue.Queue()
+ self.num_threads = num_threads
+ self.threads = []
+ self.busy = []
+ self.running = False
+ self.throttle = throttle
+ self.throttling = False
+
+ def enqueue(self, callback, *args, **kwargs):
+ if self.throttling:
+ busy_number = sum(self.busy)
+ print "Cannot add task to queue: scheduler is too busy: %d" % (busy_number,)
+ elif self.running:
+ try:
+ self.queue.put(("run", (callback, args, kwargs)), False)
+ except Queue.Full:
+ print "Cannot add task to queue: queue is full"
+ else:
+ print "Cannot add task to queue: scheduler has been shut down."
+
+ def run(self):
+
+ print "Starting threaded scheduler"
+ self.running = True
+
+ def do_work(thread_num):
+ loop = True
+ while(loop):
+ try:
+ cmd, item = self.queue.get(self.running, 10)
+ self.busy[thread_num] = 1
+ qsize = self.queue.qsize()
+ busy_number = sum(self.busy)
+ if qsize > 0:
+ print "Queue size = %d, Busy threads = %d" % (qsize, busy_number,)
+ if busy_number == self.num_threads:
+ print "*** WARNING: all threads busy"
+ if qsize >= self.num_threads * 2:
+ print "*** WARNING: throttling the task queue"
+ self.throttling = True
+
+ if cmd == "run":
+ callback, args, kwargs = item
+ try:
+ callback(*args, **kwargs)
+ except Exception, e:
+ print "Thread task exception: %s" (e,)
+ elif cmd == "stop":
+ print "Thread #%d stopping." % (thread_num,)
+ loop = False
+ self.queue.task_done()
+ self.busy[thread_num] = 0
+ busy_number = sum(self.busy)
+
+ if self.throttling and busy_number < self.num_threads/2:
+ print "*** WARNING: no longer throttling the task queue"
+ self.throttling = False
+
+ except Queue.Empty:
+ busy_number = sum(self.busy)
+ if self.throttling and busy_number < self.num_threads/2:
+ print "*** WARNING: no longer throttling the task queue"
+ self.throttling = False
+ if not self.running:
+ break
+
+ # Generate a pool of threads
+ for counter in range(self.num_threads):
+ t = Thread(target=do_work, args=(counter,))
+ t.start()
+ self.threads.append(t)
+ self.busy.append(0)
+ print "Started thread: #%d" % (counter + 1,)
+
+ print "Started threaded scheduler"
+
+ def stop(self):
+ print "Stopping threaded scheduler"
+ self.running = False
+ for ctr, thread in enumerate(self.threads):
+ thread.join()
+ print "Stopped thread: #%d" % (ctr + 1,)
+ print "Stopped threaded scheduler"
+
\ No newline at end of file
Added: CalendarClientSimulator/trunk/src/threadedsim.py
===================================================================
--- CalendarClientSimulator/trunk/src/threadedsim.py (rev 0)
+++ CalendarClientSimulator/trunk/src/threadedsim.py 2007-10-09 17:51:21 UTC (rev 1948)
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+#
+##
+# Copyright (c) 2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DRI: Cyrus Daboo, cdaboo at apple.com
+##
+
+from random import randint
+from clientscheduler import ClientScheduler
+from calendarclient import CalendarClient
+import time
+import sys
+import getopt
+
+# Simulate a whole bunch of users
+
+def usage():
+ print """Usage: threadedsim [options]
+Options:
+ --pool number of threads processing clients [10]
+ --number number of users to simulate [10]
+ --startat user number to start at [1]
+ --server URL for server (e.g. https://caldav.example.com:8443) [Required]
+ --user user id for user to login as [user%02d]
+ --password password for user [user%02d]
+ --interval polling interval in seconds [15 mins]
+ --eventsperday number of events per day to create [10]
+ --invitesperday number of invites per day to send [5]
+ --cache path to .plist file to cache data [../data/user%02d.plist]
+ --clear-cache clear the cache when starting up [Optional]
+ --no-throttle do not throttle the task queue when scheduler is too busy [Optional]
+ --verbose print out activity log
+ --logging log activity
+
+ -h, --help print this help and exit
+"""
+
+
+if __name__ == '__main__':
+
+ pool = 10
+ count = 10
+ startat = 1
+ server = None
+ user = "user%02d"
+ password = "user%02d"
+ interval = 15 * 60
+ eventsperday = 10
+ invitesperday = 5
+ cache = "../data/user%02d.plist"
+ clearcache = False
+ throttle = True
+ verbose = False
+ logging = False
+ logfile = "../logs/user%02d.txt"
+
+ options, args = getopt.getopt(sys.argv[1:], "h", [
+ "pool=",
+ "number=",
+ "startat=",
+ "server=",
+ "interval=",
+ "eventsperday=",
+ "invitesperday=",
+ "cache=",
+ "clear-cache",
+ "no-throttle",
+ "verbose",
+ "logging",
+ "help"
+ ])
+
+ for option, value in options:
+ if option in ("-h", "--help"):
+ usage()
+ sys.exit(0)
+ elif option == "--pool":
+ pool = int(value)
+ elif option == "--number":
+ count = int(value)
+ elif option == "--startat":
+ startat = int(value)
+ elif option == "--server":
+ server = value
+ elif option == "--user":
+ user = value
+ elif option == "--password":
+ password = value
+ elif option == "--interval":
+ interval = int(value)
+ elif option == "--eventsperday":
+ eventsperday = int(value)
+ elif option == "--invitesperday":
+ invitesperday = int(value)
+ elif option == "--cache":
+ cache = value
+ elif option == "--no-throttle":
+ throttle = False
+ elif option == "--verbose":
+ verbose = True
+ elif option == "--logging":
+ logging = True
+ elif option == "--clear-cache":
+ clearcache = True
+ else:
+ print "Unrecognized option: %s" % (option,)
+ usage()
+ raise ValueError
+
+ if server is None:
+ usage()
+ raise ValueError
+
+ # Create the scheduler that will manage the thread pool and client polling
+ scheduler = ClientScheduler(num_threads=pool, throttle=throttle)
+ scheduler.run()
+
+ # Create clients and add to scheduler
+ for i in range(startat, count + startat):
+ client = CalendarClient()
+ client.server = server
+ client.user = user % (i,)
+ client.password = password % (i,)
+ client.interval = interval
+ client.eventsperday = eventsperday
+ client.invitesperday = invitesperday
+ if cache:
+ client.cache = cache % (i,)
+ if clearcache:
+ client.clearcache = True
+ if verbose:
+ client.verbose = True
+ if logging:
+ def logIt(text):
+ logger = open(logfile % (i,), "a")
+ logger.write(text + "\n")
+
+ client.setLogger(logIt)
+
+ # Add random delay
+ delay = randint(1,1000)
+ time.sleep(delay/1000.0)
+
+ client.valid()
+ scheduler.add(client)
+
+ killit = raw_input("Press <RETURN> to cancel all simulations.")
+ scheduler.stop()
Property changes on: CalendarClientSimulator/trunk/src/threadedsim.py
___________________________________________________________________
Name: svn:executable
+ *
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20071009/77c726f1/attachment.html
More information about the calendarserver-changes
mailing list