[CalendarServer-changes] [15100] CalendarServer/trunk

[source_changes at macosforge.org]

Revision: 15100
          http://trac.calendarserver.org//changeset/15100
Author:   cdaboo at apple.com
Date:     2015-09-03 14:00:24 -0700 (Thu, 03 Sep 2015)
Log Message:
-----------
Improve the TEST_WORK load testing and monitoring.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/dashboard_service.py
    CalendarServer/trunk/contrib/performance/jobqueue/loadtest.py
    CalendarServer/trunk/contrib/performance/jobqueue/workrate.py
    CalendarServer/trunk/twistedcaldav/controlapi.py

Modified: CalendarServer/trunk/calendarserver/dashboard_service.py
===================================================================
--- CalendarServer/trunk/calendarserver/dashboard_service.py	2015-09-03 20:58:58 UTC (rev 15099)
+++ CalendarServer/trunk/calendarserver/dashboard_service.py	2015-09-03 21:00:24 UTC (rev 15100)
@@ -136,7 +136,7 @@
         """
 
         if self.factory.store:
-            txn = self.factory.store.newTransaction()
+            txn = self.factory.store.newTransaction("DashboardProtocol.data_jobs")
             records = (yield JobItem.histogram(txn))
             yield txn.commit()
         else:
@@ -177,10 +177,12 @@
         if self.factory.store:
             txn = self.factory.store.newTransaction()
             results["queued"] = yield TestWork.count(txn)
+            results["assigned"] = yield JobItem.count(txn, where=(JobItem.assigned != None))
             results["completed"] = WorkerConnectionPool.completed.get(TestWork.workType(), 0)
             yield txn.commit()
         else:
             results["queued"] = 0
+            results["assigned"] = 0
             results["completed"] = 0
 
         returnValue(results)

Modified: CalendarServer/trunk/contrib/performance/jobqueue/loadtest.py
===================================================================
--- CalendarServer/trunk/contrib/performance/jobqueue/loadtest.py	2015-09-03 20:58:58 UTC (rev 15099)
+++ CalendarServer/trunk/contrib/performance/jobqueue/loadtest.py	2015-09-03 21:00:24 UTC (rev 15100)
@@ -227,7 +227,7 @@
     print("Work details:")
     print("  Priority: {}").format(config["priority"])
     print("  Weight: {}").format(config["weight"])
-    print("  Start delay: {} ms").format(config["when"])
+    print("  Start delay: {} s").format(config["when"])
     print("  Execution time: {} ms").format(config["delay"])
     print("  Average queue depth: {}").format((effective_rate * config["delay"]) / 1000)
     print("")

Modified: CalendarServer/trunk/contrib/performance/jobqueue/workrate.py
===================================================================
--- CalendarServer/trunk/contrib/performance/jobqueue/workrate.py	2015-09-03 20:58:58 UTC (rev 15099)
+++ CalendarServer/trunk/contrib/performance/jobqueue/workrate.py	2015-09-03 21:00:24 UTC (rev 15100)
@@ -124,6 +124,7 @@
 
         queued = self.client.currentData["test_work"]["queued"]
         completed = self.client.currentData["test_work"]["completed"]
+        assigned = self.client.currentData["test_work"]["assigned"]
         if self.last_queued is not None:
             diff_queued = (self.last_queued - queued) / (t - self.last_time)
             diff_completed = (completed - self.last_completed) / (t - self.last_time)
@@ -133,7 +134,7 @@
         self.last_queued = queued
         self.last_completed = completed
         self.last_time = t
-        print("{}\t{:.1f}\t{:.1f}".format(queued, diff_queued, diff_completed,))
+        print("{}\t{}\t{:.1f}\t{:.1f}".format(queued, assigned, diff_queued, diff_completed,))
 
         self.sched.enter(max(self.seconds - (time.time() - t), 0), 0, self.updateResults, ())
 

Modified: CalendarServer/trunk/twistedcaldav/controlapi.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/controlapi.py	2015-09-03 20:58:58 UTC (rev 15099)
+++ CalendarServer/trunk/twistedcaldav/controlapi.py	2015-09-03 21:00:24 UTC (rev 15100)
@@ -58,6 +58,7 @@
 from txweb2.http_headers import MimeType
 
 import json
+import time
 
 log = Logger()
 
@@ -329,7 +330,7 @@
 
     @inlineCallbacks
     def action_refreshgroups(self, j):
-        txn = self._store.newTransaction()
+        txn = self._store.newTransaction(label="ControlAPIResource.action_refreshgroups")
         yield txn.directoryService().flush()
         work = yield GroupCacherPollingWork.reschedule(txn, 0, force=True)
         jobID = work.jobID
@@ -383,10 +384,12 @@
         except KeyError:
             jobs = 1
 
+        start_time = time.time()
         for _ in range(jobs):
+            effective_when = max(when - (time.time() - start_time), 0)
             yield TestWork.schedule(
                 self._store,
-                when,
+                effective_when,
                 priority,
                 weight,
                 delay,
@@ -403,7 +406,7 @@
         from txdav.common.datastore.work.revision_cleanup import _triggerRevisionCleanup
         from txdav.common.datastore.work.revision_cleanup import RevisionCleanupWork
 
-        txn = self._store.newTransaction()
+        txn = self._store.newTransaction(label="ControlAPIResource.action_revisioncleanup")
         yield _triggerRevisionCleanup(txn, 60)
         yield txn.commit()
 
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150903/89238f76/attachment-0001.html>