[CalendarServer-changes] [9532] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Mon Aug 6 14:57:17 PDT 2012
Revision: 9532
http://trac.macosforge.org/projects/calendarserver/changeset/9532
Author: cdaboo at apple.com
Date: 2012-08-06 14:57:17 -0700 (Mon, 06 Aug 2012)
Log Message:
-----------
Fix REPORTs to not depend on the size of a calendar. Use batched queries to load resources with specific names.
Allow SQL statistics results to be output to a separate file - not just error.log. Improve logic of multiget
to avoid redundant loads of resources just to see what is missing.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/twext/web2/dav/resource.py
CalendarServer/trunk/twistedcaldav/method/propfind.py
CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py
CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py
CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py
CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py
CalendarServer/trunk/twistedcaldav/resource.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/twistedcaldav/storebridge.py
CalendarServer/trunk/txdav/base/propertystore/sql.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/common/datastore/file.py
CalendarServer/trunk/txdav/common/datastore/sql.py
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -233,6 +233,7 @@
quota=quota,
logLabels=config.LogDatabase.LabelsInSQL,
logStats=config.LogDatabase.Statistics,
+ logStatsLogFile=config.LogDatabase.StatisticsLogFile,
logSQL=config.LogDatabase.SQLStatements,
logTransactionWaits=config.LogDatabase.TransactionWaitSeconds,
timeoutTransactions=config.TransactionTimeoutSeconds,
Modified: CalendarServer/trunk/twext/web2/dav/resource.py
===================================================================
--- CalendarServer/trunk/twext/web2/dav/resource.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twext/web2/dav/resource.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -681,7 +681,7 @@
@inlineCallbacks
def findChildrenFaster(
- self, depth, request, okcallback, badcallback,
+ self, depth, request, okcallback, badcallback, missingcallback,
names, privileges, inherited_aces
):
"""
@@ -699,6 +699,8 @@
that pass the privilege check, or C{None}
@param badcallback: a callback function used on all resources
that fail the privilege check, or C{None}
+ @param missingcallback: a callback function used on all resources
+ that are missing, or C{None}
@param names: a C{list} of C{str}'s containing the names of
the child resources to lookup. If empty or C{None} all
children will be examined, otherwise only the ones in the
@@ -739,6 +741,10 @@
else:
children.append((child, childpath))
+ if missingcallback:
+ for name in set(names1) - set(childnames):
+ missingcallback(joinURL(basepath, urllib.quote(name)))
+
# Generate (acl,supported_privs) map
aclmap = {}
for resource, url in children:
@@ -780,7 +786,7 @@
yield collection.inheritedACEsforChildren(request)
)
yield collection.findChildrenFaster(
- depth, request, okcallback, badcallback,
+ depth, request, okcallback, badcallback, missingcallback,
child_collections[collection_name] if names else None, privileges,
inherited_aces=collection_inherited_aces
)
Modified: CalendarServer/trunk/twistedcaldav/method/propfind.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/propfind.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/propfind.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -138,6 +138,7 @@
lambda x, y: resources.append((True, x, y)),
lambda x, y: resources.append((False, x, y)),
None,
+ None,
(davxml.Read(),),
inherited_aces=filtered_aces,
)
Modified: CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -248,6 +248,7 @@
request,
lambda x, y: ok_resources.append((x, y)),
None,
+ None,
names,
(davxml.Read(),),
inherited_aces=filteredaces
Modified: CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -207,6 +207,7 @@
request,
lambda x, y: ok_resources.append((x, y)),
None,
+ None,
names,
(davxml.Read(),),
inherited_aces=filteredaces
Modified: CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -28,10 +28,9 @@
from txdav.xml.base import dav_namespace
from twext.web2.dav.http import ErrorResponse, MultiStatusResponse
from twext.web2.dav.resource import AccessDeniedError
-from twext.web2.dav.util import joinURL
from twext.web2.http import HTTPError, StatusResponse
-from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
+from twisted.internet.defer import inlineCallbacks, returnValue
from twistedcaldav import carddavxml
from twistedcaldav.caldavxml import caldav_namespace
@@ -180,37 +179,24 @@
for href in resources:
resource_uri = str(href)
name = unquote(resource_uri[resource_uri.rfind("/") + 1:])
- child = (yield maybeDeferred(self.getChild, name))
- if not self._isChildURI(request, resource_uri) or child is None or not child.exists():
- responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+ if not self._isChildURI(request, resource_uri):
+ responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.BAD_REQUEST)))
else:
valid_names.append(name)
if not valid_names:
returnValue(None)
- # Verify that valid requested resources are calendar objects
- exists_names = tuple(
- (yield self.index().resourcesExist(valid_names))
- )
- checked_names = []
- for name in valid_names:
- if name not in exists_names:
- href = davxml.HRef.fromString(joinURL(request.uri, name))
- responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
- else:
- checked_names.append(name)
- if not checked_names:
- returnValue(None)
-
# Now determine which valid resources are readable and which are not
ok_resources = []
bad_resources = []
+ missing_resources = []
yield self.findChildrenFaster(
"1",
request,
lambda x, y: ok_resources.append((x, y)),
lambda x, y: bad_resources.append((x, y)),
- checked_names,
+ lambda x: missing_resources.append(x),
+ valid_names,
(davxml.Read(),),
inherited_aces=filteredaces
)
@@ -244,6 +230,10 @@
# Indicate error for all valid non-readable resources
for ignore_resource, href in bad_resources:
responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
+
+ # Indicate error for all missing resources
+ for href in missing_resources:
+ responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
@inlineCallbacks
def doDirectoryAddressBookResponse():
Modified: CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -128,6 +128,7 @@
request,
lambda x, y: ok_resources.append((x, y)),
lambda x, y: forbidden_resources.append((x, y)),
+ None,
changed,
(element.Read(),),
inherited_aces=filteredaces
Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/resource.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -2199,7 +2199,7 @@
@inlineCallbacks
def findChildrenFaster(
- self, depth, request, okcallback, badcallback,
+ self, depth, request, okcallback, badcallback, missingcallback,
names, privileges, inherited_aces
):
"""
@@ -2210,7 +2210,7 @@
yield self._newStoreHome.loadChildren()
result = (yield super(CommonHomeResource, self).findChildrenFaster(
- depth, request, okcallback, badcallback, names, privileges, inherited_aces
+ depth, request, okcallback, badcallback, missingcallback, names, privileges, inherited_aces
))
returnValue(result)
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -453,6 +453,7 @@
"LogDatabase" : {
"LabelsInSQL" : False,
"Statistics" : False,
+ "StatisticsLogFile" : "sqlstats.log",
"SQLStatements" : False,
"TransactionWaitSeconds" : 0,
},
@@ -1024,6 +1025,7 @@
("LogRoot", "AccessLogFile"),
("LogRoot", "ErrorLogFile"),
("LogRoot", ("Postgres", "LogFile",)),
+ ("LogRoot", ("LogDatabase", "StatisticsLogFile",)),
("LogRoot", "AccountingLogRoot"),
("RunRoot", "PIDFile"),
("RunRoot", "GlobalStatsSocket"),
Modified: CalendarServer/trunk/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/storebridge.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/storebridge.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -348,7 +348,7 @@
@inlineCallbacks
def findChildrenFaster(
- self, depth, request, okcallback, badcallback,
+ self, depth, request, okcallback, badcallback, missingcallback,
names, privileges, inherited_aces
):
"""
@@ -356,10 +356,13 @@
"""
if depth == "1":
- yield self._newStoreObject.objectResources()
+ if names:
+ yield self._newStoreObject.objectResourcesWithNames(names)
+ else:
+ yield self._newStoreObject.objectResources()
result = (yield super(_CommonHomeChildCollectionMixin, self).findChildrenFaster(
- depth, request, okcallback, badcallback, names, privileges, inherited_aces
+ depth, request, okcallback, badcallback, missingcallback, names, privileges, inherited_aces
))
returnValue(result)
Modified: CalendarServer/trunk/txdav/base/propertystore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/base/propertystore/sql.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/base/propertystore/sql.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -129,9 +129,58 @@
Where=parentColumn == parentID
)
rows = yield query.on(txn)
+ stores = cls._createMultipleStores(defaultUser, txn, rows)
+ returnValue(stores)
+
+ @classmethod
+ @inlineCallbacks
+ def forMultipleResourcesWithResourceIDs(cls, defaultUser, txn, resourceIDs):
+ """
+ Load all property stores for all specified resources. This is used
+ to optimize Depth:1 operations on that collection, by loading all
+ relevant properties in a single query. Note that the caller of this
+ method must make sure that the number of items being queried for is
+ within a reasonable batch size. If the caller is itself batching
+ related queries, that will take care of itself.
+
+ @param defaultUser: the UID of the user who owns / is requesting the
+ property stores; the ones whose per-user properties will be exposed.
+
+ @type defaultUser: C{str}
+
+ @param txn: the transaction within which to fetch the rows.
+
+ @type txn: L{IAsyncTransaction}
+
+ @param resourceIDs: The set of resource ID's to query.
+
+ @return: a L{Deferred} that fires with a C{dict} mapping resource ID (a
+ value taken from C{childColumn}) to a L{PropertyStore} for that ID.
+ """
+ query = Select([
+ prop.RESOURCE_ID, prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs)))
+ )
+ rows = yield query.on(txn, resourceIDs=resourceIDs)
+ stores = cls._createMultipleStores(defaultUser, txn, rows)
+ returnValue(stores)
+
+
+ @classmethod
+ def _createMultipleStores(cls, defaultUser, txn, rows):
+ """
+ Create a set of stores for the set of rows passed in.
+ """
+
createdStores = {}
- for object_resource_id, resource_id, name, view_uid, value in rows:
+ for row in rows:
+ if len(row) == 5:
+ object_resource_id, resource_id, name, view_uid, value = row
+ else:
+ object_resource_id = None
+ resource_id, name, view_uid, value = row
if resource_id:
if resource_id not in createdStores:
store = cls.__new__(cls)
@@ -141,7 +190,7 @@
store._cached = {}
createdStores[resource_id] = store
createdStores[resource_id]._cached[(name, view_uid)] = value
- else:
+ elif object_resource_id:
store = cls.__new__(cls)
super(PropertyStore, store).__init__(defaultUser)
store._txn = txn
@@ -149,7 +198,7 @@
store._cached = {}
createdStores[object_resource_id] = store
- returnValue(createdStores)
+ return createdStores
def _getitem_uid(self, key, uid):
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -34,7 +34,7 @@
test_event_text
from txdav.caldav.datastore.test.test_file import setUpCalendarStore
from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
-from txdav.common.datastore.sql import ECALENDARTYPE
+from txdav.common.datastore.sql import ECALENDARTYPE, CommonObjectResource
from txdav.common.datastore.sql_legacy import PostgresLegacyIndexEmulator
from txdav.common.datastore.sql_tables import schema, _BIND_MODE_DIRECT,\
_BIND_STATUS_ACCEPTED
@@ -1324,3 +1324,93 @@
yield calendar.removeCalendarObjectWithName("indexing.ics")
yield self.commit()
+
+ @inlineCallbacks
+ def test_loadObjectResourcesWithName(self):
+ """
+ L{CommonHomeChild.objectResourcesWithNames} returns the correct set of object resources
+ properly configured with a loaded property store. make sure batching works.
+ """
+
+ @inlineCallbacks
+ def _tests(cal):
+ resources = yield cal.objectResourcesWithNames(("1.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics", "3.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics", "3.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics", "3.ics", "4.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics", "3.ics", "4.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("bogus1.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set())
+
+ resources = yield cal.objectResourcesWithNames(("bogus1.ics", "2.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("2.ics",)))
+
+ # Basic load tests
+ cal = yield self.calendarUnderTest()
+ yield _tests(cal)
+
+ # Adjust batch size and try again
+ self.patch(CommonObjectResource, "BATCH_LOAD_SIZE", 2)
+ yield _tests(cal)
+
+ yield self.commit()
+
+ # Tests on inbox - resources with properties
+ txn = self.transactionUnderTest()
+ yield txn.homeWithUID(ECALENDARTYPE, "byNameTest", create=True)
+ inbox = yield self.calendarUnderTest(txn=txn, name="inbox", home="byNameTest")
+ caldata = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+METHOD:REQUEST
+BEGIN:VEVENT
+UID:instance
+DTSTART:%(now)s0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=DAILY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n") % self.nowYear
+ component = Component.fromString(caldata)
+
+ @inlineCallbacks
+ def _createInboxItem(rname, pvalue):
+ obj = yield inbox.createCalendarObjectWithName(rname, component)
+ prop = caldavxml.CalendarDescription.fromString(pvalue)
+ obj.properties()[PropertyName.fromElement(prop)] = prop
+
+ yield _createInboxItem("1.ics", "p1")
+ yield _createInboxItem("2.ics", "p2")
+ yield _createInboxItem("3.ics", "p3")
+ yield _createInboxItem("4.ics", "p4")
+ yield self.commit()
+
+ inbox = yield self.calendarUnderTest(name="inbox", home="byNameTest")
+ yield _tests(inbox)
+
+ resources = yield inbox.objectResourcesWithNames(("1.ics",))
+ prop = caldavxml.CalendarDescription.fromString("p1")
+ self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
+
+ resources = yield inbox.objectResourcesWithNames(("1.ics", "2.ics",))
+ resources.sort(key=lambda x:x._name)
+ prop = caldavxml.CalendarDescription.fromString("p1")
+ self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
+ prop = caldavxml.CalendarDescription.fromString("p2")
+ self.assertEqual(resources[1].properties()[PropertyName.fromElement(prop)], prop)
+
+ resources = yield inbox.objectResourcesWithNames(("bogus1.ics", "2.ics",))
+ resources.sort(key=lambda x:x._name)
+ prop = caldavxml.CalendarDescription.fromString("p2")
+ self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/common/datastore/file.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -800,6 +800,18 @@
for name in self.listObjectResources()]
+ def objectResourcesWithNames(self, names):
+ """
+ Return a list of the specified object resource objects.
+ """
+ results = []
+ for name in names:
+ obj = self.objectResourceWithName(name)
+ if obj is not None:
+ results.append(obj)
+ return results
+
+
def listObjectResources(self):
"""
Return a list of object resource names.
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2012-08-06 21:57:17 UTC (rev 9532)
@@ -91,7 +91,6 @@
from cStringIO import StringIO
from sqlparse import parse
import collections
-import sys
import time
current_sql_schema = getModule(__name__).filePath.sibling("sql_schema").child("current.sql").getContent()
@@ -143,7 +142,7 @@
def __init__(self, sqlTxnFactory, notifierFactory, attachmentsPath,
enableCalendars=True, enableAddressBooks=True,
label="unlabeled", quota=(2 ** 20),
- logLabels=False, logStats=False, logSQL=False,
+ logLabels=False, logStats=False, logStatsLogFile=None, logSQL=False,
logTransactionWaits=0, timeoutTransactions=0,
cacheQueries=True, cachePool="Default",
cacheExpireSeconds=3600):
@@ -158,6 +157,7 @@
self.quota = quota
self.logLabels = logLabels
self.logStats = logStats
+ self.logStatsLogFile = logStatsLogFile
self.logSQL = logSQL
self.logTransactionWaits = logTransactionWaits
self.timeoutTransactions = timeoutTransactions
@@ -220,35 +220,69 @@
class TransactionStatsCollector(object):
+ """
+ Used to log each SQL query and statistics about that query during the course of a single transaction.
+ Results can be printed out where ever needed at the end of the transaction.
+ """
- def __init__(self):
- self.count = collections.defaultdict(int)
- self.times = collections.defaultdict(float)
+ def __init__(self, label, logFileName=None):
+ self.label = label
+ self.logFileName = logFileName
+ self.statements = []
- def startStatement(self, sql):
- self.count[sql] += 1
- return sql, time.time()
+ def startStatement(self, sql, args):
+ """
+ Called prior to an SQL query being run.
- def endStatement(self, context):
- sql, tstamp = context
- self.times[sql] += time.time() - tstamp
+ @param sql: the SQL statement to execute
+ @type sql: C{str}
+ @param args: the arguments (binds) to the SQL statement
+ @type args: C{list}
- def printReport(self, toFile=sys.stdout):
+ @return: C{tuple} containing the index in the statement list for this statement, and the start time
+ """
+ args = ["%s" % (arg,) for arg in args]
+ args = [((arg[:10] + "...") if len(arg) > 40 else arg) for arg in args]
+ self.statements.append(["%s %s" % (sql, args,), 0, 0])
+ return len(self.statements) - 1, time.time()
+
+ def endStatement(self, context, rows):
+ """
+ Called after an SQL query has executed.
+
+ @param context: the tuple returned from startStatement
+ @type context: C{tuple}
+ @param rows: number of rows returned from the query
+ @type rows: C{int}
+ """
+ index, tstamp = context
+ self.statements[index][1] = len(rows) if rows else 0
+ self.statements[index][2] = time.time() - tstamp
+ def printReport(self):
+ """
+ Print a report of all the SQL statements executed to date.
+ """
+
+ toFile = StringIO()
toFile.write("*** SQL Stats ***\n")
toFile.write("\n")
- toFile.write("Unique statements: %d\n" % (len(self.count,),))
- toFile.write("Total statements: %d\n" % (sum(self.count.values()),))
- toFile.write("Total time (ms): %.3f\n" % (sum(self.times.values()) * 1000.0,))
- toFile.write("\n")
- for k, v in self.count.items():
- toFile.write("%s\n" % (k,))
- toFile.write("Count: %s\n" % (v,))
- toFile.write("Total Time (ms): %.3f\n" % (self.times[k] * 1000.0,))
- if v > 1:
- toFile.write("Average Time (ms): %.3f\n" % (self.times[k] * 1000.0 / v,))
+ toFile.write("Label: %s\n" % (self.label,))
+ toFile.write("Unique statements: %d\n" % (len(set([statement[0] for statement in self.statements]),),))
+ toFile.write("Total statements: %d\n" % (len(self.statements),))
+ toFile.write("Total rows: %d\n" % (sum([statement[1] for statement in self.statements]),))
+ toFile.write("Total time (ms): %.3f\n" % (sum([statement[2] for statement in self.statements]) * 1000.0,))
+ for sql, rows, t in self.statements:
toFile.write("\n")
- toFile.write("***\n")
+ toFile.write("SQL: %s\n" % (sql,))
+ toFile.write("Rows: %s\n" % (rows,))
+ toFile.write("Time (ms): %.3f\n" % (t,))
+ toFile.write("***\n\n")
+
+ if self.logFileName:
+ open(self.logFileName, "a").write(toFile.getvalue())
+ else:
+ log.error(toFile.getvalue())
class CommonStoreTransactionMonitor(object):
"""
@@ -349,7 +383,7 @@
self.paramstyle = sqlTxn.paramstyle
self.dialect = sqlTxn.dialect
- self._stats = TransactionStatsCollector() if self._store.logStats else None
+ self._stats = TransactionStatsCollector(self._label, self._store.logStatsLogFile) if self._store.logStats else None
self.statementCount = 0
self.iudCount = 0
self.currentStatement = None
@@ -702,7 +736,7 @@
Execute some SQL (delegate to L{IAsyncTransaction}).
"""
if self._stats:
- statsContext = self._stats.startStatement(a[0])
+ statsContext = self._stats.startStatement(a[0], a[1])
self.currentStatement = a[0]
if self._store.logTransactionWaits and a[0].split(" ", 1)[0].lower() in ("insert", "update", "delete",):
self.iudCount += 1
@@ -716,7 +750,7 @@
finally:
self.currentStatement = None
if self._stats:
- self._stats.endStatement(statsContext)
+ self._stats.endStatement(statsContext, results)
returnValue(results)
@inlineCallbacks
@@ -751,9 +785,7 @@
returnValue(ignored)
if self._stats:
- s = StringIO()
- self._stats.printReport(s)
- log.error(s.getvalue())
+ self._stats.printReport()
return self._sqlTxn.commit().addCallback(postCommit)
@@ -2717,6 +2749,19 @@
returnValue(results)
+ @inlineCallbacks
+ def objectResourcesWithNames(self, names):
+ """
+ Load and cache all named children - set of names optimization
+ """
+ results = (yield self._objectResourceClass.loadAllObjectsWithNames(self, names))
+ for result in results:
+ self._objects[result.name()] = result
+ self._objects[result.uid()] = result
+ self._objectNames = sorted([result.name() for result in results])
+ returnValue(results)
+
+
@classproperty
def _objectResourceNamesQuery(cls): #@NoSelf
"""
@@ -3100,6 +3145,8 @@
_objectSchema = None
+ BATCH_LOAD_SIZE = 50
+
def __init__(self, parent, name, uid, resourceID=None, metadata=None):
self._parentCollection = parent
self._resourceID = resourceID
@@ -3161,8 +3208,74 @@
returnValue(results)
+ @classmethod
+ def _allColumnsWithParentAndNames(cls, names): #@NoSelf
+ obj = cls._objectSchema
+ return Select(cls._allColumns, From=obj,
+ Where=(obj.PARENT_RESOURCE_ID == Parameter("parentID")).And(
+ obj.RESOURCE_NAME.In(Parameter("names", len(names)))))
+
@classmethod
+ @inlineCallbacks
+ def loadAllObjectsWithNames(cls, parent, names):
+ """
+ Load all child objects with the specified names, doing so in batches.
+ """
+ names = tuple(names)
+ results = []
+ while(len(names)):
+ result_batch = (yield cls._loadAllObjectsWithNames(parent, names[:cls.BATCH_LOAD_SIZE]))
+ results.extend(result_batch)
+ names = names[cls.BATCH_LOAD_SIZE:]
+
+ returnValue(results)
+
+ @classmethod
+ @inlineCallbacks
+ def _loadAllObjectsWithNames(cls, parent, names):
+ """
+ Load all child objects with the specified names. This must create the
+ child classes and initialize them using "batched" SQL operations to keep
+ this constant wrt the number of children. This is an optimization for
+ Depth:1 operations on the collection.
+ """
+
+ # Optimize case of single name to load
+ if len(names) == 1:
+ obj = yield cls.objectWithName(parent, names[0], None)
+ returnValue([obj] if obj else [])
+
+ results = []
+
+ # Load from the main table first
+ dataRows = yield cls._allColumnsWithParentAndNames(names).on(
+ parent._txn, parentID=parent._resourceID, names=names)
+
+ if dataRows:
+ # Get property stores for all these child resources
+ if parent.objectResourcesHaveProperties():
+ propertyStores =(yield PropertyStore.forMultipleResourcesWithResourceIDs(
+ parent._home.uid(),
+ parent._txn,
+ tuple([row[0] for row in dataRows]),
+ ))
+ else:
+ propertyStores = {}
+
+ # Create the actual objects merging in properties
+ for row in dataRows:
+ child = cls(parent, "", None)
+ child._initFromRow(tuple(row))
+ yield child._loadPropertyStore(
+ props=propertyStores.get(child._resourceID, None)
+ )
+ results.append(child)
+
+ returnValue(results)
+
+
+ @classmethod
def objectWithName(cls, parent, name, uid):
objectResource = cls(parent, name, uid, None)
return objectResource.initFromStore()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120806/e32e56ee/attachment-0001.html>
More information about the calendarserver-changes
mailing list