[CalendarServer-changes] [9532] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Mon Aug 6 14:57:17 PDT 2012


Revision: 9532
          http://trac.macosforge.org/projects/calendarserver/changeset/9532
Author:   cdaboo at apple.com
Date:     2012-08-06 14:57:17 -0700 (Mon, 06 Aug 2012)
Log Message:
-----------
Fix REPORTs to not depend on the size of a calendar. Use batched queries to load resources with specific names.
Allow SQL statistics results to be output to a separate file - not just error.log. Improve logic of multiget
to avoid redundant loads of resources just to see what is missing.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/twext/web2/dav/resource.py
    CalendarServer/trunk/twistedcaldav/method/propfind.py
    CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py
    CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py
    CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py
    CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py
    CalendarServer/trunk/twistedcaldav/resource.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/twistedcaldav/storebridge.py
    CalendarServer/trunk/txdav/base/propertystore/sql.py
    CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
    CalendarServer/trunk/txdav/common/datastore/file.py
    CalendarServer/trunk/txdav/common/datastore/sql.py

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -233,6 +233,7 @@
             quota=quota,
             logLabels=config.LogDatabase.LabelsInSQL,
             logStats=config.LogDatabase.Statistics,
+            logStatsLogFile=config.LogDatabase.StatisticsLogFile,
             logSQL=config.LogDatabase.SQLStatements,
             logTransactionWaits=config.LogDatabase.TransactionWaitSeconds,
             timeoutTransactions=config.TransactionTimeoutSeconds,

Modified: CalendarServer/trunk/twext/web2/dav/resource.py
===================================================================
--- CalendarServer/trunk/twext/web2/dav/resource.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twext/web2/dav/resource.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -681,7 +681,7 @@
 
     @inlineCallbacks
     def findChildrenFaster(
-        self, depth, request, okcallback, badcallback,
+        self, depth, request, okcallback, badcallback, missingcallback,
         names, privileges, inherited_aces
     ):
         """
@@ -699,6 +699,8 @@
             that pass the privilege check, or C{None}
         @param badcallback: a callback function used on all resources
             that fail the privilege check, or C{None}
+        @param missingcallback: a callback function used on all resources
+            that are missing, or C{None}
         @param names: a C{list} of C{str}'s containing the names of
             the child resources to lookup. If empty or C{None} all
             children will be examined, otherwise only the ones in the
@@ -739,6 +741,10 @@
                 else:
                     children.append((child, childpath))
 
+        if missingcallback:
+            for name in set(names1) - set(childnames):
+                missingcallback(joinURL(basepath, urllib.quote(name)))
+
         # Generate (acl,supported_privs) map
         aclmap = {}
         for resource, url in children:
@@ -780,7 +786,7 @@
                         yield collection.inheritedACEsforChildren(request)
                     )
                     yield collection.findChildrenFaster(
-                        depth, request, okcallback, badcallback,
+                        depth, request, okcallback, badcallback, missingcallback,
                         child_collections[collection_name] if names else None, privileges,
                         inherited_aces=collection_inherited_aces
                     )

Modified: CalendarServer/trunk/twistedcaldav/method/propfind.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/propfind.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/propfind.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -138,6 +138,7 @@
         lambda x, y: resources.append((True, x, y)),
         lambda x, y: resources.append((False, x, y)),
         None,
+        None,
         (davxml.Read(),),
         inherited_aces=filtered_aces,
     )

Modified: CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_addressbook_query.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -248,6 +248,7 @@
                             request,
                             lambda x, y: ok_resources.append((x, y)),
                             None,
+                            None,
                             names,
                             (davxml.Read(),),
                             inherited_aces=filteredaces

Modified: CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_calendar_query.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -207,6 +207,7 @@
                     request,
                     lambda x, y: ok_resources.append((x, y)),
                     None,
+                    None,
                     names,
                     (davxml.Read(),),
                     inherited_aces=filteredaces

Modified: CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_multiget_common.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -28,10 +28,9 @@
 from txdav.xml.base import dav_namespace
 from twext.web2.dav.http import ErrorResponse, MultiStatusResponse
 from twext.web2.dav.resource import AccessDeniedError
-from twext.web2.dav.util import joinURL
 from twext.web2.http import HTTPError, StatusResponse
 
-from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
+from twisted.internet.defer import inlineCallbacks, returnValue
 
 from twistedcaldav import carddavxml
 from twistedcaldav.caldavxml import caldav_namespace
@@ -180,37 +179,24 @@
             for href in resources:
                 resource_uri = str(href)
                 name = unquote(resource_uri[resource_uri.rfind("/") + 1:])
-                child = (yield maybeDeferred(self.getChild, name))
-                if not self._isChildURI(request, resource_uri) or child is None or not child.exists():
-                    responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+                if not self._isChildURI(request, resource_uri):
+                    responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.BAD_REQUEST)))
                 else:
                     valid_names.append(name)
             if not valid_names:
                 returnValue(None)
         
-            # Verify that valid requested resources are calendar objects
-            exists_names = tuple(
-                (yield self.index().resourcesExist(valid_names))
-            )
-            checked_names = []
-            for name in valid_names:
-                if name not in exists_names:
-                    href = davxml.HRef.fromString(joinURL(request.uri, name))
-                    responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
-                else:
-                    checked_names.append(name)
-            if not checked_names:
-                returnValue(None)
-            
             # Now determine which valid resources are readable and which are not
             ok_resources = []
             bad_resources = []
+            missing_resources = []
             yield self.findChildrenFaster(
                 "1",
                 request,
                 lambda x, y: ok_resources.append((x, y)),
                 lambda x, y: bad_resources.append((x, y)),
-                checked_names,
+                lambda x: missing_resources.append(x),
+                valid_names,
                 (davxml.Read(),),
                 inherited_aces=filteredaces
             )
@@ -244,6 +230,10 @@
             # Indicate error for all valid non-readable resources
             for ignore_resource, href in bad_resources:
                 responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
+
+            # Indicate error for all missing resources
+            for href in missing_resources:
+                responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
     
         @inlineCallbacks
         def doDirectoryAddressBookResponse():

Modified: CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -128,6 +128,7 @@
             request,
             lambda x, y: ok_resources.append((x, y)),
             lambda x, y: forbidden_resources.append((x, y)),
+            None,
             changed,
             (element.Read(),),
             inherited_aces=filteredaces

Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/resource.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -2199,7 +2199,7 @@
 
     @inlineCallbacks
     def findChildrenFaster(
-        self, depth, request, okcallback, badcallback,
+        self, depth, request, okcallback, badcallback, missingcallback,
         names, privileges, inherited_aces
     ):
         """
@@ -2210,7 +2210,7 @@
             yield self._newStoreHome.loadChildren()
         
         result = (yield super(CommonHomeResource, self).findChildrenFaster(
-            depth, request, okcallback, badcallback, names, privileges, inherited_aces
+            depth, request, okcallback, badcallback, missingcallback, names, privileges, inherited_aces
         ))
         
         returnValue(result)

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -453,6 +453,7 @@
     "LogDatabase" : {
         "LabelsInSQL"            : False,
         "Statistics"             : False,
+        "StatisticsLogFile"      : "sqlstats.log",
         "SQLStatements"          : False,
         "TransactionWaitSeconds" : 0, 
     },
@@ -1024,6 +1025,7 @@
     ("LogRoot", "AccessLogFile"),
     ("LogRoot", "ErrorLogFile"),
     ("LogRoot", ("Postgres", "LogFile",)),
+    ("LogRoot", ("LogDatabase", "StatisticsLogFile",)),
     ("LogRoot", "AccountingLogRoot"),
     ("RunRoot", "PIDFile"),
     ("RunRoot", "GlobalStatsSocket"),

Modified: CalendarServer/trunk/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/storebridge.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/twistedcaldav/storebridge.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -348,7 +348,7 @@
 
     @inlineCallbacks
     def findChildrenFaster(
-        self, depth, request, okcallback, badcallback,
+        self, depth, request, okcallback, badcallback, missingcallback,
         names, privileges, inherited_aces
     ):
         """
@@ -356,10 +356,13 @@
         """
         
         if depth == "1":
-            yield self._newStoreObject.objectResources()
+            if names:
+                yield self._newStoreObject.objectResourcesWithNames(names)
+            else:
+                yield self._newStoreObject.objectResources()
         
         result = (yield super(_CommonHomeChildCollectionMixin, self).findChildrenFaster(
-            depth, request, okcallback, badcallback, names, privileges, inherited_aces
+            depth, request, okcallback, badcallback, missingcallback, names, privileges, inherited_aces
         ))
         
         returnValue(result)

Modified: CalendarServer/trunk/txdav/base/propertystore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/base/propertystore/sql.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/base/propertystore/sql.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -129,9 +129,58 @@
             Where=parentColumn == parentID
         )
         rows = yield query.on(txn)
+        stores = cls._createMultipleStores(defaultUser, txn, rows)
+        returnValue(stores)
 
+
+    @classmethod
+    @inlineCallbacks
+    def forMultipleResourcesWithResourceIDs(cls, defaultUser, txn, resourceIDs):
+        """
+        Load all property stores for all specified resources.  This is used
+        to optimize Depth:1 operations on that collection, by loading all
+        relevant properties in a single query. Note that the caller of this
+        method must make sure that the number of items being queried for is
+        within a reasonable batch size. If the caller is itself batching
+        related queries, that will take care of itself.
+
+        @param defaultUser: the UID of the user who owns / is requesting the
+            property stores; the ones whose per-user properties will be exposed.
+
+        @type defaultUser: C{str}
+
+        @param txn: the transaction within which to fetch the rows.
+
+        @type txn: L{IAsyncTransaction}
+
+        @param resourceIDs: The set of resource ID's to query.
+
+        @return: a L{Deferred} that fires with a C{dict} mapping resource ID (a
+            value taken from C{childColumn}) to a L{PropertyStore} for that ID.
+        """
+        query = Select([
+            prop.RESOURCE_ID, prop.NAME, prop.VIEWER_UID, prop.VALUE],
+            From=prop,
+            Where=prop.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs)))
+        )
+        rows = yield query.on(txn, resourceIDs=resourceIDs)
+        stores = cls._createMultipleStores(defaultUser, txn, rows)
+        returnValue(stores)
+
+
+    @classmethod
+    def _createMultipleStores(cls, defaultUser, txn, rows):
+        """
+        Create a set of stores for the set of rows passed in.
+        """
+
         createdStores = {}
-        for object_resource_id, resource_id, name, view_uid, value in rows:
+        for row in rows:
+            if len(row) == 5:
+                object_resource_id, resource_id, name, view_uid, value = row
+            else:
+                object_resource_id = None
+                resource_id, name, view_uid, value = row
             if resource_id:
                 if resource_id not in createdStores:
                     store = cls.__new__(cls)
@@ -141,7 +190,7 @@
                     store._cached = {}
                     createdStores[resource_id] = store
                 createdStores[resource_id]._cached[(name, view_uid)] = value
-            else:
+            elif object_resource_id:
                 store = cls.__new__(cls)
                 super(PropertyStore, store).__init__(defaultUser)
                 store._txn = txn
@@ -149,7 +198,7 @@
                 store._cached = {}
                 createdStores[object_resource_id] = store
 
-        returnValue(createdStores)
+        return createdStores
 
 
     def _getitem_uid(self, key, uid):

Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -34,7 +34,7 @@
     test_event_text
 from txdav.caldav.datastore.test.test_file import setUpCalendarStore
 from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
-from txdav.common.datastore.sql import ECALENDARTYPE
+from txdav.common.datastore.sql import ECALENDARTYPE, CommonObjectResource
 from txdav.common.datastore.sql_legacy import PostgresLegacyIndexEmulator
 from txdav.common.datastore.sql_tables import schema, _BIND_MODE_DIRECT,\
     _BIND_STATUS_ACCEPTED
@@ -1324,3 +1324,93 @@
         
         yield calendar.removeCalendarObjectWithName("indexing.ics")
         yield self.commit()
+
+    @inlineCallbacks
+    def test_loadObjectResourcesWithName(self):
+        """
+        L{CommonHomeChild.objectResourcesWithNames} returns the correct set of object resources
+        properly configured with a loaded property store. make sure batching works.
+        """
+
+        @inlineCallbacks
+        def _tests(cal):
+            resources = yield cal.objectResourcesWithNames(("1.ics",))
+            self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics",)))
+    
+            resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics",))
+            self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics",)))
+    
+            resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics", "3.ics",))
+            self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics", "3.ics",)))
+    
+            resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics", "3.ics", "4.ics",))
+            self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics", "3.ics", "4.ics",)))
+    
+            resources = yield cal.objectResourcesWithNames(("bogus1.ics",))
+            self.assertEqual(set([resource.name() for resource in resources]), set())
+    
+            resources = yield cal.objectResourcesWithNames(("bogus1.ics", "2.ics",))
+            self.assertEqual(set([resource.name() for resource in resources]), set(("2.ics",)))
+
+        # Basic load tests
+        cal = yield self.calendarUnderTest()
+        yield _tests(cal)
+
+        # Adjust batch size and try again
+        self.patch(CommonObjectResource, "BATCH_LOAD_SIZE", 2)
+        yield _tests(cal)
+        
+        yield self.commit()
+
+        # Tests on inbox - resources with properties
+        txn = self.transactionUnderTest()
+        yield txn.homeWithUID(ECALENDARTYPE, "byNameTest", create=True)
+        inbox = yield self.calendarUnderTest(txn=txn, name="inbox", home="byNameTest")
+        caldata = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+METHOD:REQUEST
+BEGIN:VEVENT
+UID:instance
+DTSTART:%(now)s0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=DAILY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n") % self.nowYear
+        component = Component.fromString(caldata)
+
+        @inlineCallbacks
+        def _createInboxItem(rname, pvalue):
+            obj = yield inbox.createCalendarObjectWithName(rname, component)
+            prop = caldavxml.CalendarDescription.fromString(pvalue)
+            obj.properties()[PropertyName.fromElement(prop)] = prop
+
+        yield _createInboxItem("1.ics", "p1")
+        yield _createInboxItem("2.ics", "p2")
+        yield _createInboxItem("3.ics", "p3")
+        yield _createInboxItem("4.ics", "p4")
+        yield self.commit()
+
+        inbox = yield self.calendarUnderTest(name="inbox", home="byNameTest")
+        yield _tests(inbox)
+
+        resources = yield inbox.objectResourcesWithNames(("1.ics",))
+        prop = caldavxml.CalendarDescription.fromString("p1")
+        self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
+
+        resources = yield inbox.objectResourcesWithNames(("1.ics", "2.ics",))
+        resources.sort(key=lambda x:x._name)
+        prop = caldavxml.CalendarDescription.fromString("p1")
+        self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
+        prop = caldavxml.CalendarDescription.fromString("p2")
+        self.assertEqual(resources[1].properties()[PropertyName.fromElement(prop)], prop)
+
+        resources = yield inbox.objectResourcesWithNames(("bogus1.ics", "2.ics",))
+        resources.sort(key=lambda x:x._name)
+        prop = caldavxml.CalendarDescription.fromString("p2")
+        self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)

Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/common/datastore/file.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -800,6 +800,18 @@
                 for name in self.listObjectResources()]
 
 
+    def objectResourcesWithNames(self, names):
+        """
+        Return a list of the specified object resource objects.
+        """
+        results = []
+        for name in names:
+            obj = self.objectResourceWithName(name)
+            if obj is not None:
+                results.append(obj)
+        return results
+
+
     def listObjectResources(self):
         """
         Return a list of object resource names.

Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py	2012-08-06 21:39:46 UTC (rev 9531)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py	2012-08-06 21:57:17 UTC (rev 9532)
@@ -91,7 +91,6 @@
 from cStringIO import StringIO
 from sqlparse import parse
 import collections
-import sys
 import time
 
 current_sql_schema = getModule(__name__).filePath.sibling("sql_schema").child("current.sql").getContent()
@@ -143,7 +142,7 @@
     def __init__(self, sqlTxnFactory, notifierFactory, attachmentsPath,
                  enableCalendars=True, enableAddressBooks=True,
                  label="unlabeled", quota=(2 ** 20),
-                 logLabels=False, logStats=False, logSQL=False,
+                 logLabels=False, logStats=False, logStatsLogFile=None, logSQL=False,
                  logTransactionWaits=0, timeoutTransactions=0,
                  cacheQueries=True, cachePool="Default",
                  cacheExpireSeconds=3600):
@@ -158,6 +157,7 @@
         self.quota = quota
         self.logLabels = logLabels
         self.logStats = logStats
+        self.logStatsLogFile = logStatsLogFile
         self.logSQL = logSQL
         self.logTransactionWaits = logTransactionWaits
         self.timeoutTransactions = timeoutTransactions
@@ -220,35 +220,69 @@
 
 
 class TransactionStatsCollector(object):
+    """
+    Used to log each SQL query and statistics about that query during the course of a single transaction.
+    Results can be printed out where ever needed at the end of the transaction.
+    """
     
-    def __init__(self):
-        self.count = collections.defaultdict(int)
-        self.times = collections.defaultdict(float)
+    def __init__(self, label, logFileName=None):
+        self.label = label
+        self.logFileName = logFileName
+        self.statements = []
     
-    def startStatement(self, sql):
-        self.count[sql] += 1
-        return sql, time.time()
+    def startStatement(self, sql, args):
+        """
+        Called prior to an SQL query being run.
 
-    def endStatement(self, context):
-        sql, tstamp = context
-        self.times[sql] += time.time() - tstamp
+        @param sql: the SQL statement to execute
+        @type sql: C{str}
+        @param args: the arguments (binds) to the SQL statement
+        @type args: C{list}
         
-    def printReport(self, toFile=sys.stdout):
+        @return: C{tuple} containing the index in the statement list for this statement, and the start time
+        """
+        args = ["%s" % (arg,) for arg in args]
+        args = [((arg[:10] + "...") if len(arg) > 40 else arg) for arg in args]
+        self.statements.append(["%s %s" % (sql, args,), 0, 0])
+        return len(self.statements) - 1, time.time()
+
+    def endStatement(self, context, rows):
+        """
+        Called after an SQL query has executed.
+
+        @param context: the tuple returned from startStatement
+        @type context: C{tuple}
+        @param rows: number of rows returned from the query
+        @type rows: C{int}
+        """
+        index, tstamp = context
+        self.statements[index][1] = len(rows) if rows else 0
+        self.statements[index][2] = time.time() - tstamp
         
+    def printReport(self):
+        """
+        Print a report of all the SQL statements executed to date.
+        """
+        
+        toFile = StringIO()
         toFile.write("*** SQL Stats ***\n")
         toFile.write("\n")
-        toFile.write("Unique statements: %d\n" % (len(self.count,),))
-        toFile.write("Total statements: %d\n" % (sum(self.count.values()),))
-        toFile.write("Total time (ms): %.3f\n" % (sum(self.times.values()) * 1000.0,))
-        toFile.write("\n")
-        for k, v in self.count.items():
-            toFile.write("%s\n" % (k,))
-            toFile.write("Count: %s\n" % (v,))
-            toFile.write("Total Time (ms): %.3f\n" % (self.times[k] * 1000.0,))
-            if v > 1:
-                toFile.write("Average Time (ms): %.3f\n" % (self.times[k] * 1000.0 / v,))
+        toFile.write("Label: %s\n" % (self.label,))
+        toFile.write("Unique statements: %d\n" % (len(set([statement[0] for statement in self.statements]),),))
+        toFile.write("Total statements: %d\n" % (len(self.statements),))
+        toFile.write("Total rows: %d\n" % (sum([statement[1] for statement in self.statements]),))
+        toFile.write("Total time (ms): %.3f\n" % (sum([statement[2] for statement in self.statements]) * 1000.0,))
+        for sql, rows, t in self.statements:
             toFile.write("\n")
-        toFile.write("***\n")
+            toFile.write("SQL: %s\n" % (sql,))
+            toFile.write("Rows: %s\n" % (rows,))
+            toFile.write("Time (ms): %.3f\n" % (t,))
+        toFile.write("***\n\n")
+        
+        if self.logFileName:
+            open(self.logFileName, "a").write(toFile.getvalue())
+        else:
+            log.error(toFile.getvalue())
 
 class CommonStoreTransactionMonitor(object):
     """
@@ -349,7 +383,7 @@
         self.paramstyle = sqlTxn.paramstyle
         self.dialect = sqlTxn.dialect
 
-        self._stats = TransactionStatsCollector() if self._store.logStats else None
+        self._stats = TransactionStatsCollector(self._label, self._store.logStatsLogFile) if self._store.logStats else None
         self.statementCount = 0
         self.iudCount = 0
         self.currentStatement = None
@@ -702,7 +736,7 @@
         Execute some SQL (delegate to L{IAsyncTransaction}).
         """
         if self._stats:        
-            statsContext = self._stats.startStatement(a[0])
+            statsContext = self._stats.startStatement(a[0], a[1])
         self.currentStatement = a[0]
         if self._store.logTransactionWaits and a[0].split(" ", 1)[0].lower() in ("insert", "update", "delete",):
             self.iudCount += 1
@@ -716,7 +750,7 @@
         finally:
             self.currentStatement = None
             if self._stats:        
-                self._stats.endStatement(statsContext)
+                self._stats.endStatement(statsContext, results)
         returnValue(results)
 
     @inlineCallbacks
@@ -751,9 +785,7 @@
             returnValue(ignored)
 
         if self._stats:
-            s = StringIO()
-            self._stats.printReport(s)
-            log.error(s.getvalue())
+            self._stats.printReport()
 
         return self._sqlTxn.commit().addCallback(postCommit)
 
@@ -2717,6 +2749,19 @@
         returnValue(results)
 
 
+    @inlineCallbacks
+    def objectResourcesWithNames(self, names):
+        """
+        Load and cache all named children - set of names optimization
+        """
+        results = (yield self._objectResourceClass.loadAllObjectsWithNames(self, names))
+        for result in results:
+            self._objects[result.name()] = result
+            self._objects[result.uid()] = result
+        self._objectNames = sorted([result.name() for result in results])
+        returnValue(results)
+
+
     @classproperty
     def _objectResourceNamesQuery(cls): #@NoSelf
         """
@@ -3100,6 +3145,8 @@
 
     _objectSchema = None
 
+    BATCH_LOAD_SIZE = 50
+
     def __init__(self, parent, name, uid, resourceID=None, metadata=None):
         self._parentCollection = parent
         self._resourceID = resourceID
@@ -3161,8 +3208,74 @@
 
         returnValue(results)
 
+    @classmethod
+    def _allColumnsWithParentAndNames(cls, names): #@NoSelf
+        obj = cls._objectSchema
+        return Select(cls._allColumns, From=obj,
+                      Where=(obj.PARENT_RESOURCE_ID == Parameter("parentID")).And(
+                          obj.RESOURCE_NAME.In(Parameter("names", len(names)))))
 
+
     @classmethod
+    @inlineCallbacks
+    def loadAllObjectsWithNames(cls, parent, names):
+        """
+        Load all child objects with the specified names, doing so in batches.
+        """
+        names = tuple(names)
+        results = []
+        while(len(names)):
+            result_batch = (yield cls._loadAllObjectsWithNames(parent, names[:cls.BATCH_LOAD_SIZE]))
+            results.extend(result_batch)
+            names = names[cls.BATCH_LOAD_SIZE:]
+        
+        returnValue(results)
+            
+    @classmethod
+    @inlineCallbacks
+    def _loadAllObjectsWithNames(cls, parent, names):
+        """
+        Load all child objects with the specified names. This must create the
+        child classes and initialize them using "batched" SQL operations to keep
+        this constant wrt the number of children. This is an optimization for
+        Depth:1 operations on the collection.
+        """
+
+        # Optimize case of single name to load
+        if len(names) == 1:
+            obj = yield cls.objectWithName(parent, names[0], None)
+            returnValue([obj] if obj else [])
+
+        results = []
+
+        # Load from the main table first
+        dataRows = yield cls._allColumnsWithParentAndNames(names).on(
+            parent._txn, parentID=parent._resourceID, names=names)
+
+        if dataRows:
+            # Get property stores for all these child resources
+            if parent.objectResourcesHaveProperties():
+                propertyStores =(yield PropertyStore.forMultipleResourcesWithResourceIDs(
+                    parent._home.uid(),
+                    parent._txn,
+                    tuple([row[0] for row in dataRows]),
+                ))
+            else:
+                propertyStores = {}
+
+        # Create the actual objects merging in properties
+        for row in dataRows:
+            child = cls(parent, "", None)
+            child._initFromRow(tuple(row))
+            yield child._loadPropertyStore(
+                props=propertyStores.get(child._resourceID, None)
+            )
+            results.append(child)
+
+        returnValue(results)
+
+
+    @classmethod
     def objectWithName(cls, parent, name, uid):
         objectResource = cls(parent, name, uid, None)
         return objectResource.initFromStore()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120806/e32e56ee/attachment-0001.html>


More information about the calendarserver-changes mailing list