[CalendarServer-changes] [10156] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Dec 11 13:35:08 PST 2012
Revision: 10156
http://trac.calendarserver.org//changeset/10156
Author: cdaboo at apple.com
Date: 2012-12-11 13:35:08 -0800 (Tue, 11 Dec 2012)
Log Message:
-----------
Print a more detailed table of orphaned attachments in dry-run mode.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/purge.py
CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/trunk/twext/enterprise/dal/syntax.py
CalendarServer/trunk/txdav/common/datastore/sql.py
Modified: CalendarServer/trunk/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/purge.py 2012-12-11 21:05:45 UTC (rev 10155)
+++ CalendarServer/trunk/calendarserver/tools/purge.py 2012-12-11 21:35:08 UTC (rev 10156)
@@ -44,7 +44,10 @@
from calendarserver.tools.cmdline import utilityMain
from calendarserver.tools.principals import removeProxy
+from calendarserver.tools import tables
+import collections
+
log = Logger()
DEFAULT_BATCH_SIZE = 100
@@ -61,7 +64,7 @@
print " -h --help: print this help and exit"
print " -f --config <path>: Specify caldavd.plist configuration path"
print " -d --days <number>: specify how many days in the past to retain (default=%d)" % (DEFAULT_RETAIN_DAYS,)
- #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
print " -n --dry-run: calculate how many events to purge, but do not purge data"
print " -v --verbose: print progress information"
print ""
@@ -72,6 +75,8 @@
else:
sys.exit(0)
+
+
def usage_purge_orphaned_attachments(e=None):
name = os.path.basename(sys.argv[0])
@@ -82,7 +87,7 @@
print "options:"
print " -h --help: print this help and exit"
print " -f --config <path>: Specify caldavd.plist configuration path"
- #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
print " -n --dry-run: calculate how many attachments to purge, but do not purge data"
print " -v --verbose: print progress information"
print ""
@@ -93,6 +98,8 @@
else:
sys.exit(0)
+
+
def usage_purge_principal(e=None):
name = os.path.basename(sys.argv[0])
@@ -115,11 +122,13 @@
sys.exit(0)
+
class WorkerService(Service):
def __init__(self, store):
self._store = store
+
def rootResource(self):
try:
rootResource = getRootResource(config, self._store)
@@ -209,7 +218,6 @@
-
def main_purge_events():
try:
@@ -285,6 +293,7 @@
)
+
def main_purge_orphaned_attachments():
try:
@@ -347,6 +356,7 @@
)
+
def main_purge_principals():
try:
@@ -401,13 +411,13 @@
PurgePrincipalService.verbose = verbose
PurgePrincipalService.doimplicit = doimplicit
-
utilityMain(
configFileName,
PurgePrincipalService
)
+
@inlineCallbacks
def purgeOldEvents(store, directory, root, date, batchSize, verbose=False,
dryrun=False):
@@ -462,16 +472,42 @@
print "(Dry run) Searching for orphaned attachments..."
txn = store.newTransaction(label="Find orphaned attachments")
orphans = (yield txn.orphanedAttachments())
- orphanCount = len(orphans)
if verbose:
- if orphanCount == 0:
- print "No orphaned attachments"
- elif orphanCount == 1:
- print "1 orphaned attachment"
- else:
- print "%d orphaned attachments" % (orphanCount,)
- returnValue(orphanCount)
+ # Print aggregate details by user
+ byuser = collections.defaultdict(int)
+ for owner_uid, _ignore_dropbox_id, _ignore_path, size in orphans:
+ byuser[owner_uid] += size
+ # Print table of results
+ table = tables.Table()
+ table.addHeader(("User", "Current Quota", "Orphaned Size", "Orphaned Count"))
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+ total = 0
+ for user, quota, size, count in sorted(orphans):
+ table.addRow((
+ user,
+ quota,
+ size,
+ count,
+ ))
+ total += count
+ table.addFooter(("Total:", "", "", total))
+
+ print "\n"
+ print "Orphaned Attachments by User:\n"
+ table.printTable()
+ else:
+ total = sum([x[3] for x in orphans])
+
+ returnValue(total)
+
if verbose:
print "Removing orphaned attachments..."
@@ -499,14 +535,12 @@
-
-
@inlineCallbacks
def purgeUIDs(store, directory, root, uids, verbose=False, dryrun=False,
completely=False, doimplicit=True):
total = 0
- allAssignments = { }
+ allAssignments = {}
for uid in uids:
count, allAssignments[uid] = (yield purgeUID(store, uid, directory, root,
@@ -607,7 +641,6 @@
main.removeProperty(exdate_rdate)
dirty = True
-
# Remove any overridden components beyond the cutoff
for component in tuple(event.subcomponents()):
if component.name() == "VEVENT":
@@ -625,6 +658,7 @@
return CANCELEVENT_NOT_MODIFIED
+
@inlineCallbacks
def purgeUID(store, uid, directory, root, verbose=False, dryrun=False, proxies=True,
when=None, completely=False, doimplicit=True):
@@ -687,7 +721,7 @@
# Anything in the past is left alone
whenString = when.getText()
- filter = caldavxml.Filter(
+ filter = caldavxml.Filter(
caldavxml.ComponentFilter(
caldavxml.ComponentFilter(
TimeRange(start=whenString,),
@@ -718,7 +752,7 @@
childNames.append(childName)
else:
# events matching filter
- for childName, childUid, childType in (yield collection.index().indexedSearch(filter)):
+ for childName, _ignore_childUid, _ignore_childType in (yield collection.index().indexedSearch(filter)):
childNames.append(childName)
for childName in childNames:
@@ -792,7 +826,6 @@
if incrementCount:
count += 1
-
txn = getattr(request, "_newStoreTransaction", None)
# Commit
if txn is not None:
@@ -835,12 +868,11 @@
if not dryrun:
(yield storeCalHome.remove())
-
# Remove VCards
storeAbHome = (yield txn.addressbookHomeWithUID(uid))
if storeAbHome is not None:
- for abColl in list( (yield storeAbHome.addressbooks()) ):
- for card in list( (yield abColl.addressbookObjects()) ):
+ for abColl in list((yield storeAbHome.addressbooks())):
+ for card in list((yield abColl.addressbookObjects())):
cardName = card.name()
if verbose:
uri = "/addressbooks/__uids__/%s/%s/%s" % (uid, abColl.name(), cardName)
@@ -885,6 +917,7 @@
returnValue((count, assignments))
+
@inlineCallbacks
def purgeProxyAssignments(principal):
@@ -905,4 +938,3 @@
(yield subPrincipal.writeProperty(davxml.GroupMemberSet(), None))
returnValue(assignments)
-
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2012-12-11 21:05:45 UTC (rev 10155)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2012-12-11 21:35:08 UTC (rev 10156)
@@ -84,7 +84,7 @@
SEQUENCE:2
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year":now-5}
+""".replace("\n", "\r\n") % {"year": now - 5}
OLD_ATTACHMENT_ICS = """BEGIN:VCALENDAR
VERSION:2.0
@@ -134,7 +134,7 @@
SEQUENCE:2
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year":now-5}
+""".replace("\n", "\r\n") % {"year": now - 5}
ENDLESS_ICS = """BEGIN:VCALENDAR
VERSION:2.0
@@ -183,7 +183,7 @@
SEQUENCE:4
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year":now-5}
+""".replace("\n", "\r\n") % {"year": now - 5}
REPEATING_AWHILE_ICS = """BEGIN:VCALENDAR
VERSION:2.0
@@ -232,7 +232,7 @@
SEQUENCE:6
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year":now-5}
+""".replace("\n", "\r\n") % {"year": now - 5}
STRADDLING_ICS = """BEGIN:VCALENDAR
VERSION:2.0
@@ -267,7 +267,7 @@
SEQUENCE:5
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year":now-2, "until":now+1}
+""".replace("\n", "\r\n") % {"year": now - 2, "until": now + 1}
RECENT_ICS = """BEGIN:VCALENDAR
VERSION:2.0
@@ -301,7 +301,7 @@
SEQUENCE:2
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year":now}
+""".replace("\n", "\r\n") % {"year": now}
VCARD_1 = """BEGIN:VCARD
@@ -421,7 +421,7 @@
count = (yield txn.removeOldEvents(cutoff))
self.assertEquals(count, 3)
results = (yield txn.eventsOlderThan(cutoff))
- self.assertEquals(results, [ ])
+ self.assertEquals(results, [])
# Remove oldest events (none left)
count = (yield txn.removeOldEvents(cutoff))
@@ -470,6 +470,7 @@
# Just look for orphaned attachments but don't delete
orphans = (yield txn.orphanedAttachments())
self.assertEquals(len(orphans), 1)
+ self.assertEquals(orphans, [["home1", 19, 19, 1]])
# Remove orphaned attachments, should be 1
count = (yield txn.removeOrphanedAttachments(batchSize=100))
@@ -515,12 +516,12 @@
abColl = (yield abHome.addressbookWithName("addressbook"))
(yield abColl.createAddressBookObjectWithName("card1",
VCardComponent.fromString(VCARD_1)))
- self.assertEquals(len( (yield abColl.addressbookObjects()) ), 1)
+ self.assertEquals(len((yield abColl.addressbookObjects())), 1)
# Verify there are 3 events in calendar1
calHome = (yield txn.calendarHomeWithUID("home1"))
calColl = (yield calHome.calendarWithName("calendar1"))
- self.assertEquals(len( (yield calColl.calendarObjects()) ), 3)
+ self.assertEquals(len((yield calColl.calendarObjects())), 3)
# Make the newly created objects available to the purgeUID transaction
(yield txn.commit())
@@ -540,7 +541,7 @@
calHome = (yield txn.calendarHomeWithUID("home1"))
calColl = (yield calHome.calendarWithName("calendar1"))
- self.assertEquals(len( (yield calColl.calendarObjects()) ), 2)
+ self.assertEquals(len((yield calColl.calendarObjects())), 2)
@inlineCallbacks
@@ -552,12 +553,12 @@
abColl = (yield abHome.addressbookWithName("addressbook"))
(yield abColl.createAddressBookObjectWithName("card1",
VCardComponent.fromString(VCARD_1)))
- self.assertEquals(len( (yield abColl.addressbookObjects()) ), 1)
+ self.assertEquals(len((yield abColl.addressbookObjects())), 1)
# Verify there are 3 events in calendar1
calHome = (yield txn.calendarHomeWithUID("home1"))
calColl = (yield calHome.calendarWithName("calendar1"))
- self.assertEquals(len( (yield calColl.calendarObjects()) ), 3)
+ self.assertEquals(len((yield calColl.calendarObjects())), 3)
# Make the newly created objects available to the purgeUID transaction
(yield txn.commit())
@@ -601,4 +602,3 @@
total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
dryrun=False, verbose=False))
self.assertEquals(total, 0)
-
Modified: CalendarServer/trunk/twext/enterprise/dal/syntax.py
===================================================================
--- CalendarServer/trunk/twext/enterprise/dal/syntax.py 2012-12-11 21:05:45 UTC (rev 10155)
+++ CalendarServer/trunk/twext/enterprise/dal/syntax.py 2012-12-11 21:35:08 UTC (rev 10156)
@@ -41,6 +41,8 @@
except ImportError:
cx_Oracle = None
+
+
class DALError(Exception):
"""
Base class for exceptions raised by this module. This can be raised
@@ -327,8 +329,8 @@
# 0'.)
__add__ = comparison("+")
__sub__ = comparison("-")
- __div__= comparison("/")
- __mul__= comparison("*")
+ __div__ = comparison("/")
+ __mul__ = comparison("*")
def __nonzero__(self):
@@ -365,6 +367,8 @@
def Contains(self, other):
return CompoundComparison(self, "like", CompoundComparison(Constant('%'), '||', CompoundComparison(Constant(other), '||', Constant('%'))))
+
+
class FunctionInvocation(ExpressionSyntax):
def __init__(self, function, *args):
self.function = function
@@ -453,6 +457,7 @@
Count = Function("count")
+Sum = Function("sum")
Max = Function("max")
Len = Function("character_length", "length")
Upper = Function("upper")
@@ -780,23 +785,28 @@
return self.model.table.name + '.' + name
+
class ResultAliasSyntax(ExpressionSyntax):
def __init__(self, expression, alias=None):
self.expression = expression
self.alias = alias
+
def aliasName(self, queryGenerator):
if self.alias is None:
self.alias = queryGenerator.nextGeneratedID()
return self.alias
+
def columnReference(self):
return AliasReferenceSyntax(self)
+
def allColumns(self):
return self.expression.allColumns()
+
def subSQL(self, queryGenerator, allTables):
result = SQLFragment()
result.append(self.expression.subSQL(queryGenerator, allTables))
@@ -804,18 +814,22 @@
return result
+
class AliasReferenceSyntax(ExpressionSyntax):
def __init__(self, resultAlias):
self.resultAlias = resultAlias
+
def allColumns(self):
return self.resultAlias.allColumns()
+
def subSQL(self, queryGenerator, allTables):
return SQLFragment(self.resultAlias.aliasName(queryGenerator))
+
class AliasedColumnSyntax(ColumnSyntax):
"""
An L{AliasedColumnSyntax} is like a L{ColumnSyntax}, but it generates SQL
@@ -898,9 +912,9 @@
def subSQL(self, queryGenerator, allTables):
- if ( queryGenerator.dialect == ORACLE_DIALECT
+ if (queryGenerator.dialect == ORACLE_DIALECT
and isinstance(self.b, Constant) and self.b.value == ''
- and self.op in ('=', '!=') ):
+ and self.op in ('=', '!=')):
return NullComparison(self.a, self.op).subSQL(queryGenerator, allTables)
stmt = SQLFragment()
result = self._subexpression(self.a, queryGenerator, allTables)
@@ -948,6 +962,7 @@
def __init__(self):
self.name = "*"
+
def allColumns(self):
return []
@@ -1024,6 +1039,7 @@
return self.columns
+
class SetExpression(object):
"""
A UNION, INTERSECT, or EXCEPT construct used inside a SELECT.
@@ -1052,6 +1068,7 @@
if self.optype not in (None, SetExpression.OPTYPE_ALL, SetExpression.OPTYPE_DISTINCT,):
raise DALError("Must have either 'all' or 'distinct' in a set expression")
+
def subSQL(self, queryGenerator, allTables):
result = SQLFragment()
for select in self.selects:
@@ -1063,9 +1080,12 @@
result.append(select.subSQL(queryGenerator, allTables))
return result
+
def allColumns(self):
return []
+
+
class Union(SetExpression):
"""
A UNION construct used inside a SELECT.
@@ -1073,6 +1093,8 @@
def setOpSQL(self, queryGenerator):
return SQLFragment(" UNION ")
+
+
class Intersect(SetExpression):
"""
An INTERSECT construct used inside a SELECT.
@@ -1080,6 +1102,8 @@
def setOpSQL(self, queryGenerator):
return SQLFragment(" INTERSECT ")
+
+
class Except(SetExpression):
"""
An EXCEPT construct used inside a SELECT.
@@ -1092,6 +1116,8 @@
else:
raise NotImplementedError("Unsupported dialect")
+
+
class Select(_Statement):
"""
'select' statement.
@@ -1131,6 +1157,7 @@
if self.From.As is None:
self.From.As = ""
+
def __eq__(self, other):
"""
Create a comparison.
@@ -1238,6 +1265,7 @@
for column in self.columns.columns:
yield column
+
def tables(self):
"""
Determine the tables used by the result columns.
@@ -1255,6 +1283,7 @@
return [TableSyntax(table) for table in tables]
+
def _commaJoined(stmts):
first = True
cstatement = SQLFragment()
@@ -1663,6 +1692,7 @@
return SQLFragment('savepoint %s' % (self.name,))
+
class RollbackToSavepoint(_LockingStatement):
"""
An SQL 'rollback to savepoint' statement.
@@ -1676,6 +1706,7 @@
return SQLFragment('rollback to savepoint %s' % (self.name,))
+
class ReleaseSavepoint(_LockingStatement):
"""
An SQL 'release savepoint' statement.
@@ -1821,4 +1852,3 @@
# (Although this is a special keyword in a CREATE statement, in an INSERT it
# behaves like an expression to the best of my knowledge.)
default = NamedValue('default')
-
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2012-12-11 21:05:45 UTC (rev 10155)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2012-12-11 21:35:08 UTC (rev 10156)
@@ -74,7 +74,7 @@
from twext.enterprise.dal.syntax import \
Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction, \
- Select, Update, ColumnSyntax, TableSyntax, Upper, Count, ALL_COLUMNS
+ Select, Update, ColumnSyntax, TableSyntax, Upper, Count, ALL_COLUMNS, Sum
from twistedcaldav.config import config
@@ -969,39 +969,63 @@
returnValue(count)
- def _orphanedBase(limited): #@NoSelf
+ def _orphanedSummary(limited): #@NoSelf
at = schema.ATTACHMENT
co = schema.CALENDAR_OBJECT
+ ch = schema.CALENDAR_HOME
+ chm = schema.CALENDAR_HOME_METADATA
kwds = {}
if limited:
kwds["Limit"] = Parameter('batchSize')
return Select(
- [at.DROPBOX_ID, at.PATH],
- From=at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer"),
+ [ch.OWNER_UID, chm.QUOTA_USED_BYTES, Sum(at.SIZE), Count(at.DROPBOX_ID)],
+ From=at.join(
+ co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer").join(
+ ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID).join(
+ chm, ch.RESOURCE_ID == chm.RESOURCE_ID
+ ),
Where=co.DROPBOX_ID == None,
+ GroupBy=(ch.OWNER_UID, chm.QUOTA_USED_BYTES),
**kwds
)
- _orphanedLimited = _orphanedBase(True)
- _orphanedUnlimited = _orphanedBase(False)
- del _orphanedBase
+ _orphanedSummaryLimited = _orphanedSummary(True)
+ _orphanedSummaryUnlimited = _orphanedSummary(False)
+ del _orphanedSummary
-
def orphanedAttachments(self, batchSize=None):
"""
Find attachments no longer referenced by any events.
- Returns a deferred to a list of (dropbox_id, path) tuples.
+ Returns a deferred to a list of (calendar_home_owner_uid, dropbox_id, path, size) tuples.
"""
if batchSize is not None:
kwds = {'batchSize': batchSize}
- query = self._orphanedLimited
+ query = self._orphanedSummaryLimited
else:
kwds = {}
- query = self._orphanedUnlimited
+ query = self._orphanedSummaryUnlimited
return query.on(self, **kwds)
+ def _orphanedBase(limited): #@NoSelf
+ at = schema.ATTACHMENT
+ co = schema.CALENDAR_OBJECT
+ kwds = {}
+ if limited:
+ kwds["Limit"] = Parameter('batchSize')
+ return Select(
+ [at.DROPBOX_ID, at.PATH],
+ From=at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer"),
+ Where=co.DROPBOX_ID == None,
+ **kwds
+ )
+
+ _orphanedLimited = _orphanedBase(True)
+ _orphanedUnlimited = _orphanedBase(False)
+ del _orphanedBase
+
+
@inlineCallbacks
def removeOrphanedAttachments(self, batchSize=None):
"""
@@ -1011,7 +1035,13 @@
# TODO: see if there is a better way to import Attachment
from txdav.caldav.datastore.sql import DropBoxAttachment
- results = (yield self.orphanedAttachments(batchSize=batchSize))
+ if batchSize is not None:
+ kwds = {'batchSize': batchSize}
+ query = self._orphanedLimited
+ else:
+ kwds = {}
+ query = self._orphanedUnlimited
+ results = (yield query.on(self, **kwds))
count = 0
for dropboxID, path in results:
attachment = (yield DropBoxAttachment.load(self, dropboxID, path))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121211/dd7e7660/attachment-0001.html>
More information about the calendarserver-changes
mailing list