[CalendarServer-changes] [14532] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Mon Mar 9 12:47:39 PDT 2015
Revision: 14532
http://trac.calendarserver.org//changeset/14532
Author: cdaboo at apple.com
Date: 2015-03-09 12:47:39 -0700 (Mon, 09 Mar 2015)
Log Message:
-----------
Support for pg8000 (not currently enabled for use).
Modified Paths:
--------------
CalendarServer/trunk/.project
CalendarServer/trunk/calendarserver/tools/calverify.py
CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/trunk/requirements-stable.txt
CalendarServer/trunk/twistedcaldav/dateops.py
CalendarServer/trunk/twistedcaldav/test/test_dateops.py
CalendarServer/trunk/txdav/base/datastore/subpostgres.py
CalendarServer/trunk/txdav/caldav/datastore/index_file.py
CalendarServer/trunk/txdav/caldav/datastore/query/builder.py
CalendarServer/trunk/txdav/caldav/datastore/query/test/test_filter.py
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/caldav/datastore/sql_attachment.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/carddav/datastore/sql.py
CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/sql_notification.py
CalendarServer/trunk/txdav/common/datastore/sql_tables.py
CalendarServer/trunk/txdav/common/datastore/test/test_sql_tables.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade_with_data.py
CalendarServer/trunk/txdav/who/test/test_group_attendees.py
Modified: CalendarServer/trunk/.project
===================================================================
--- CalendarServer/trunk/.project 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/.project 2015-03-09 19:47:39 UTC (rev 14532)
@@ -6,6 +6,7 @@
<project>CalDAVClientLibrary</project>
<project>cffi</project>
<project>kerberos</project>
+ <project>pg8000</project>
<project>psutil</project>
<project>pycalendar</project>
<project>pycparser</project>
Modified: CalendarServer/trunk/calendarserver/tools/calverify.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/calverify.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/calendarserver/tools/calverify.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -60,7 +60,7 @@
from twisted.python import usage
from twisted.python.usage import Options
from twistedcaldav.datafilters.peruserdata import PerUserDataFilter
-from twistedcaldav.dateops import pyCalendarTodatetime
+from twistedcaldav.dateops import pyCalendarToSQLTimestamp
from twistedcaldav.ical import Component, InvalidICalendarDataError, Property, PERUSER_COMPONENT
from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
from twistedcaldav.timezones import TimezoneCache
@@ -530,8 +530,8 @@
ch = schema.CALENDAR_HOME
tr = schema.TIME_RANGE
kwds = {
- "Start" : pyCalendarTodatetime(start),
- "Max" : pyCalendarTodatetime(DateTime(1900, 1, 1, 0, 0, 0))
+ "Start" : pyCalendarToSQLTimestamp(start),
+ "Max" : pyCalendarToSQLTimestamp(DateTime(1900, 1, 1, 0, 0, 0))
}
rows = (yield Select(
[ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
@@ -583,8 +583,8 @@
ch = schema.CALENDAR_HOME
tr = schema.TIME_RANGE
kwds = {
- "Start" : pyCalendarTodatetime(start),
- "Max" : pyCalendarTodatetime(DateTime(1900, 1, 1, 0, 0, 0)),
+ "Start" : pyCalendarToSQLTimestamp(start),
+ "Max" : pyCalendarToSQLTimestamp(DateTime(1900, 1, 1, 0, 0, 0)),
"UUID" : uuid,
}
rows = (yield Select(
@@ -613,8 +613,8 @@
cb.CALENDAR_RESOURCE_NAME != "inbox")
kwds = {
- "Start" : pyCalendarTodatetime(start),
- "Max" : pyCalendarTodatetime(DateTime(1900, 1, 1, 0, 0, 0)),
+ "Start" : pyCalendarToSQLTimestamp(start),
+ "Max" : pyCalendarToSQLTimestamp(DateTime(1900, 1, 1, 0, 0, 0)),
"UUID" : uuid,
}
rows = (yield Select(
@@ -2159,7 +2159,7 @@
tr = schema.TIME_RANGE
kwds = {
"uuid": uuid,
- "Start" : pyCalendarTodatetime(start),
+ "Start" : pyCalendarToSQLTimestamp(start),
}
rows = (yield Select(
[co.RESOURCE_ID, ],
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -25,6 +25,7 @@
)
from pycalendar.datetime import DateTime
from twext.enterprise.dal.syntax import Update, Delete
+from twext.enterprise.util import parseSQLTimestamp
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
from twistedcaldav.config import config
@@ -458,19 +459,19 @@
self.assertEquals(
sorted(results),
sorted([
- ['home1', 'calendar1', 'old.ics', '1901-01-01 01:00:00'],
- ['home1', 'calendar1', 'oldattachment1.ics', '1901-01-01 01:00:00'],
- ['home1', 'calendar1', 'oldattachment2.ics', '1901-01-01 01:00:00'],
- ['home1', 'calendar1', 'oldmattachment1.ics', '1901-01-01 01:00:00'],
- ['home1', 'calendar1', 'oldmattachment2.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar3', 'repeating_awhile.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar2', 'recent.ics', '%s-03-04 22:15:00' % (now,)],
- ['home2', 'calendar2', 'oldattachment1.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar2', 'oldattachment3.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar2', 'oldattachment4.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar2', 'oldmattachment1.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar2', 'oldmattachment3.ics', '1901-01-01 01:00:00'],
- ['home2', 'calendar2', 'oldmattachment4.ics', '1901-01-01 01:00:00'],
+ ['home1', 'calendar1', 'old.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home1', 'calendar1', 'oldattachment1.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home1', 'calendar1', 'oldattachment2.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home1', 'calendar1', 'oldmattachment1.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home1', 'calendar1', 'oldmattachment2.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar3', 'repeating_awhile.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar2', 'recent.ics', parseSQLTimestamp('%s-03-04 22:15:00' % (now,))],
+ ['home2', 'calendar2', 'oldattachment1.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar2', 'oldattachment3.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar2', 'oldattachment4.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar2', 'oldmattachment1.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar2', 'oldmattachment3.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
+ ['home2', 'calendar2', 'oldmattachment4.ics', parseSQLTimestamp('1901-01-01 01:00:00')],
])
)
@@ -497,7 +498,7 @@
count = (yield txn.removeOldEvents(cutoff))
self.assertEquals(count, 12)
results = (yield txn.eventsOlderThan(cutoff))
- self.assertEquals(results, [])
+ self.assertEquals(list(results), [])
# Remove oldest events (none left)
count = (yield txn.removeOldEvents(cutoff))
Modified: CalendarServer/trunk/requirements-stable.txt
===================================================================
--- CalendarServer/trunk/requirements-stable.txt 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/requirements-stable.txt 2015-03-09 19:47:39 UTC (rev 14532)
@@ -36,7 +36,7 @@
#pyOpenSSL
pycrypto==2.6.1
- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14521#egg=twextpy
+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14531#egg=twextpy
cffi==0.8.6
pycparser==2.10
#twisted
Modified: CalendarServer/trunk/twistedcaldav/dateops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/dateops.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/twistedcaldav/dateops.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -32,7 +32,7 @@
from pycalendar.timezone import Timezone
from pycalendar.period import Period
-import datetime
+from datetime import date, datetime
import dateutil.tz
import calendar
@@ -269,19 +269,19 @@
-def pyCalendarTodatetime(pydt):
+def pyCalendarToSQLTimestamp(pydt):
if pydt.isDateOnly():
- return datetime.date(year=pydt.getYear(), month=pydt.getMonth(), day=pydt.getDay())
+ return date(year=pydt.getYear(), month=pydt.getMonth(), day=pydt.getDay())
else:
- return datetime.datetime(
+ return datetime(
year=pydt.getYear(),
month=pydt.getMonth(),
day=pydt.getDay(),
hour=pydt.getHours(),
minute=pydt.getMinutes(),
second=pydt.getSeconds(),
- tzinfo=dateutil.tz.tzutc()
+ tzinfo=None
)
@@ -295,15 +295,25 @@
@return: L{DateTime} result
"""
- # Format is "%Y-%m-%d %H:%M:%S"
- return DateTime(
- year=int(ts[0:4]),
- month=int(ts[5:7]),
- day=int(ts[8:10]),
- hours=int(ts[11:13]),
- minutes=int(ts[14:16]),
- seconds=int(ts[17:19])
- )
+ if isinstance(ts, datetime):
+ return DateTime(
+ year=ts.year,
+ month=ts.month,
+ day=ts.day,
+ hours=ts.hour,
+ minutes=ts.minute,
+ seconds=ts.second
+ )
+ else:
+ # Format is "%Y-%m-%d %H:%M:%S"
+ return DateTime(
+ year=int(ts[0:4]),
+ month=int(ts[5:7]),
+ day=int(ts[8:10]),
+ hours=int(ts[11:13]),
+ minutes=int(ts[14:16]),
+ seconds=int(ts[17:19])
+ )
@@ -316,18 +326,25 @@
@return: L{DateTime} result
"""
- # Format is "%Y-%m-%d", though Oracle may add zero time which we ignore
- return DateTime(
- year=int(ts[0:4]),
- month=int(ts[5:7]),
- day=int(ts[8:10])
- )
+ if isinstance(ts, date):
+ return DateTime(
+ year=ts.year,
+ month=ts.month,
+ day=ts.day,
+ )
+ else:
+ # Format is "%Y-%m-%d", though Oracle may add zero time which we ignore
+ return DateTime(
+ year=int(ts[0:4]),
+ month=int(ts[5:7]),
+ day=int(ts[8:10])
+ )
def datetimeMktime(dt):
- assert isinstance(dt, datetime.date)
+ assert isinstance(dt, date)
if dt.tzinfo is None:
dt.replace(tzinfo=dateutil.tz.tzutc())
Modified: CalendarServer/trunk/twistedcaldav/test/test_dateops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_dateops.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/twistedcaldav/test/test_dateops.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -19,11 +19,10 @@
from pycalendar.datetime import DateTime
from twistedcaldav.dateops import parseSQLTimestampToPyCalendar, \
- parseSQLDateToPyCalendar, pyCalendarTodatetime, \
+ parseSQLDateToPyCalendar, pyCalendarToSQLTimestamp, \
normalizeForExpand, normalizeForIndex, normalizeToUTC, timeRangesOverlap
-import datetime
-import dateutil
+from datetime import datetime, date
from pycalendar.timezone import Timezone
from twistedcaldav.timezones import TimezoneCache
@@ -249,17 +248,17 @@
raise SkipTest("test unimplemented")
- def test_pyCalendarTodatetime(self):
+ def test_pyCalendarToSQLTimestamp(self):
"""
- dateops.pyCalendarTodatetime
+ dateops.pyCalendarToSQLTimestamp
"""
tests = (
- (DateTime(2012, 4, 4, 12, 34, 56), datetime.datetime(2012, 4, 4, 12, 34, 56, tzinfo=dateutil.tz.tzutc())),
- (DateTime(2012, 12, 31), datetime.date(2012, 12, 31)),
+ (DateTime(2012, 4, 4, 12, 34, 56), datetime(2012, 4, 4, 12, 34, 56, tzinfo=None)),
+ (DateTime(2012, 12, 31), date(2012, 12, 31)),
)
for pycal, result in tests:
- self.assertEqual(pyCalendarTodatetime(pycal), result)
+ self.assertEqual(pyCalendarToSQLTimestamp(pycal), result)
def test_parseSQLTimestampToPyCalendar(self):
@@ -269,6 +268,8 @@
tests = (
("2012-04-04 12:34:56", DateTime(2012, 4, 4, 12, 34, 56)),
("2012-12-31 01:01:01", DateTime(2012, 12, 31, 1, 1, 1)),
+ (datetime(2012, 4, 4, 12, 34, 56), DateTime(2012, 4, 4, 12, 34, 56)),
+ (datetime(2012, 12, 31, 1, 1, 1), DateTime(2012, 12, 31, 1, 1, 1)),
)
for sqlStr, result in tests:
@@ -283,6 +284,8 @@
tests = (
("2012-04-04", DateTime(2012, 4, 4)),
("2012-12-31 00:00:00", DateTime(2012, 12, 31)),
+ (date(2012, 4, 4), DateTime(2012, 4, 4)),
+ (date(2012, 12, 31), DateTime(2012, 12, 31)),
)
for sqlStr, result in tests:
Modified: CalendarServer/trunk/txdav/base/datastore/subpostgres.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -26,11 +26,15 @@
from hashlib import md5
from pipes import quote as shell_quote
-import pgdb as postgres
+if True:
+ import pgdb as postgres
+else:
+ import pg8000 as postgres
from twisted.python.procutils import which
from twisted.internet.protocol import ProcessProtocol
+from twext.enterprise.dal.parseschema import splitSQLString
from twext.python.log import Logger
from twext.python.filepath import CachingFilePath
@@ -38,7 +42,6 @@
from twisted.internet.defer import Deferred
from txdav.base.datastore.dbapiclient import DBAPIConnector
from txdav.base.datastore.dbapiclient import postgresPreflight
-from txdav.common.datastore.sql_tables import splitSQLString
from txdav.common.icommondatastore import InternalDataStoreError
from twisted.application.service import MultiService
@@ -388,9 +391,30 @@
"""
Produce a DB-API 2.0 connection pointed at this database.
"""
- return self._connectorFor(databaseName).connect(label)
+ connection = self._connectorFor(databaseName).connect(label)
+ if postgres.__name__ == "pg8000":
+ # Patch pg8000 behavior to match what we need wrt text processing
+ def my_text_out(v):
+ return v.encode("utf-8") if isinstance(v, unicode) else str(v)
+ connection.realConnection.py_types[str] = (705, postgres.core.FC_TEXT, my_text_out)
+ connection.realConnection.py_types[postgres.six.text_type] = (705, postgres.core.FC_TEXT, my_text_out)
+
+ def my_text_recv(data, offset, length):
+ return str(data[offset: offset + length])
+ connection.realConnection.default_factory = lambda: (postgres.core.FC_TEXT, my_text_recv)
+ connection.realConnection.pg_types[19] = (postgres.core.FC_BINARY, my_text_recv)
+ connection.realConnection.pg_types[25] = (postgres.core.FC_BINARY, my_text_recv)
+ connection.realConnection.pg_types[705] = (postgres.core.FC_BINARY, my_text_recv)
+ connection.realConnection.pg_types[829] = (postgres.core.FC_TEXT, my_text_recv)
+ connection.realConnection.pg_types[1042] = (postgres.core.FC_BINARY, my_text_recv)
+ connection.realConnection.pg_types[1043] = (postgres.core.FC_BINARY, my_text_recv)
+ connection.realConnection.pg_types[2275] = (postgres.core.FC_BINARY, my_text_recv)
+
+ return connection
+
+
def ready(self, createDatabaseConn, createDatabaseCursor):
"""
Subprocess is ready. Time to initialize the subservice.
Modified: CalendarServer/trunk/txdav/caldav/datastore/index_file.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/index_file.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/index_file.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -50,7 +50,7 @@
from txdav.common.icommondatastore import SyncTokenValidException, \
ReservationError, IndexedSearchException
-from twistedcaldav.dateops import pyCalendarTodatetime
+from twistedcaldav.dateops import pyCalendarToSQLTimestamp
from twistedcaldav.ical import Component
from twistedcaldav.sql import AbstractSQLDatabase
from twistedcaldav.sql import db_prefix
@@ -658,7 +658,7 @@
Gives all resources which have not been expanded beyond a given date
in the index
"""
- return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX < :1", pyCalendarTodatetime(minDate))
+ return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX < :1", pyCalendarToSQLTimestamp(minDate))
def reExpandResource(self, name, expand_until):
@@ -747,7 +747,7 @@
"""
insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX, ORGANIZER)
values (:1, :2, :3, :4, :5)
- """, name, uid, calendar.resourceType(), pyCalendarTodatetime(recurrenceLimit) if recurrenceLimit else None, organizer
+ """, name, uid, calendar.resourceType(), pyCalendarToSQLTimestamp(recurrenceLimit) if recurrenceLimit else None, organizer
)
resourceid = self.lastrowid
@@ -785,8 +785,8 @@
""",
resourceid,
float,
- pyCalendarTodatetime(start),
- pyCalendarTodatetime(end),
+ pyCalendarToSQLTimestamp(start),
+ pyCalendarToSQLTimestamp(end),
icalfbtype_to_indexfbtype.get(instance.component.getFBType(), 'F'),
transp
)
@@ -811,7 +811,7 @@
"""
insert into TIMESPAN (RESOURCEID, FLOAT, START, END, FBTYPE, TRANSPARENT)
values (:1, :2, :3, :4, :5, :6)
- """, resourceid, float, pyCalendarTodatetime(start), pyCalendarTodatetime(end), '?', '?'
+ """, resourceid, float, pyCalendarToSQLTimestamp(start), pyCalendarToSQLTimestamp(end), '?', '?'
)
instanceid = self.lastrowid
peruserdata = calendar.perUserData(None)
Modified: CalendarServer/trunk/txdav/caldav/datastore/query/builder.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/query/builder.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/query/builder.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -14,7 +14,7 @@
# limitations under the License.
##
-from twistedcaldav.dateops import floatoffset, pyCalendarTodatetime
+from twistedcaldav.dateops import floatoffset, pyCalendarToSQLTimestamp
from txdav.caldav.datastore.query.filter import ComponentFilter, PropertyFilter, TextMatch, TimeRange
from txdav.common.datastore.query import expression
@@ -220,8 +220,8 @@
endfloat = floatoffset(end, tzinfo) if end else None
return (
- pyCalendarTodatetime(start) if start else None,
- pyCalendarTodatetime(end) if end else None,
- pyCalendarTodatetime(startfloat) if startfloat else None,
- pyCalendarTodatetime(endfloat) if endfloat else None,
+ pyCalendarToSQLTimestamp(start) if start else None,
+ pyCalendarToSQLTimestamp(end) if end else None,
+ pyCalendarToSQLTimestamp(startfloat) if startfloat else None,
+ pyCalendarToSQLTimestamp(endfloat) if endfloat else None,
)
Modified: CalendarServer/trunk/txdav/caldav/datastore/query/test/test_filter.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/query/test/test_filter.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/query/test/test_filter.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -29,7 +29,6 @@
from txdav.caldav.datastore.query.generator import CalDAVSQLQueryGenerator
from txdav.common.datastore.sql_tables import schema
-from dateutil.tz import tzutc
import datetime
from twistedcaldav.ical import Component
@@ -97,7 +96,7 @@
self.assertEqual(select.toSQL(), SQLFragment(
"select distinct RESOURCE_NAME, ICALENDAR_UID, ICALENDAR_TYPE from CALENDAR_OBJECT, TIME_RANGE where ICALENDAR_TYPE in (?, ?, ?) and (FLOATING = ? and START_DATE < ? and END_DATE > ? or FLOATING = ? and START_DATE < ? and END_DATE > ?) and CALENDAR_OBJECT_RESOURCE_ID = RESOURCE_ID and TIME_RANGE.CALENDAR_RESOURCE_ID = ?",
- [Parameter('arg1', 3), False, datetime.datetime(2006, 6, 5, 17, 0, tzinfo=tzutc()), datetime.datetime(2006, 6, 5, 16, 0, tzinfo=tzutc()), True, datetime.datetime(2006, 6, 5, 13, 0, tzinfo=tzutc()), datetime.datetime(2006, 6, 5, 12, 0, tzinfo=tzutc()), 1234]
+ [Parameter('arg1', 3), False, datetime.datetime(2006, 6, 5, 17, 0), datetime.datetime(2006, 6, 5, 16, 0), True, datetime.datetime(2006, 6, 5, 13, 0), datetime.datetime(2006, 6, 5, 12, 0), 1234]
))
self.assertEqual(args, {"arg1": ("VEVENT", "VFREEBUSY", "VAVAILABILITY")})
self.assertEqual(usedtimerange, True)
@@ -126,7 +125,7 @@
self.assertEqual(select.toSQL(), SQLFragment(
"select distinct RESOURCE_NAME, ICALENDAR_UID, ICALENDAR_TYPE, ORGANIZER, FLOATING, coalesce(ADJUSTED_START_DATE, START_DATE), coalesce(ADJUSTED_END_DATE, END_DATE), FBTYPE, TIME_RANGE.TRANSPARENT, PERUSER.TRANSPARENT from CALENDAR_OBJECT, TIME_RANGE left outer join PERUSER on INSTANCE_ID = TIME_RANGE_INSTANCE_ID and USER_ID = ? where ICALENDAR_TYPE in (?, ?, ?) and (FLOATING = ? and coalesce(ADJUSTED_START_DATE, START_DATE) < ? and coalesce(ADJUSTED_END_DATE, END_DATE) > ? or FLOATING = ? and coalesce(ADJUSTED_START_DATE, START_DATE) < ? and coalesce(ADJUSTED_END_DATE, END_DATE) > ?) and CALENDAR_OBJECT_RESOURCE_ID = RESOURCE_ID and TIME_RANGE.CALENDAR_RESOURCE_ID = ?",
- ['user01', Parameter('arg1', 3), False, datetime.datetime(2006, 6, 5, 17, 0, tzinfo=tzutc()), datetime.datetime(2006, 6, 5, 16, 0, tzinfo=tzutc()), True, datetime.datetime(2006, 6, 5, 13, 0, tzinfo=tzutc()), datetime.datetime(2006, 6, 5, 12, 0, tzinfo=tzutc()), 1234]
+ ['user01', Parameter('arg1', 3), False, datetime.datetime(2006, 6, 5, 17, 0), datetime.datetime(2006, 6, 5, 16, 0), True, datetime.datetime(2006, 6, 5, 13, 0), datetime.datetime(2006, 6, 5, 12, 0), 1234]
))
self.assertEqual(args, {"arg1": ("VEVENT", "VFREEBUSY", "VAVAILABILITY")})
self.assertEqual(usedtimerange, True)
@@ -193,7 +192,7 @@
self.assertEqual(select.toSQL(), SQLFragment(
"select distinct RESOURCE_NAME, ICALENDAR_UID, ICALENDAR_TYPE from CALENDAR_OBJECT, TIME_RANGE where (ICALENDAR_TYPE = ? and (FLOATING = ? and END_DATE > ? or FLOATING = ? and END_DATE > ?) or ICALENDAR_TYPE = ?) and CALENDAR_OBJECT_RESOURCE_ID = RESOURCE_ID and TIME_RANGE.CALENDAR_RESOURCE_ID = ?",
- ['VEVENT', False, datetime.datetime(2006, 6, 5, 16, 0, tzinfo=tzutc()), True, datetime.datetime(2006, 6, 5, 12, 0, tzinfo=tzutc()), 'VTODO', 1234]
+ ['VEVENT', False, datetime.datetime(2006, 6, 5, 16, 0), True, datetime.datetime(2006, 6, 5, 12, 0), 'VTODO', 1234]
))
self.assertEqual(args, {})
self.assertEqual(usedtimerange, True)
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
+from twext.enterprise.util import parseSQLTimestamp
"""
@@ -47,7 +48,7 @@
from twistedcaldav.stdconfig import config
from twistedcaldav.datafilters.peruserdata import PerUserDataFilter
from twistedcaldav.dateops import normalizeForIndex, \
- pyCalendarTodatetime, parseSQLDateToPyCalendar
+ pyCalendarToSQLTimestamp, parseSQLDateToPyCalendar
from twistedcaldav.ical import Component, InvalidICalendarDataError, Property
from twistedcaldav.instance import InvalidOverriddenInstanceError
from twistedcaldav.timezones import TimezoneException
@@ -1622,8 +1623,8 @@
returnValue([row[0] for row in (
yield self._notExpandedWithinQuery.on(
self._txn,
- minDate=pyCalendarTodatetime(normalizeForIndex(minDate)) if minDate is not None else None,
- maxDate=pyCalendarTodatetime(normalizeForIndex(maxDate)),
+ minDate=pyCalendarToSQLTimestamp(normalizeForIndex(minDate)) if minDate is not None else None,
+ maxDate=pyCalendarToSQLTimestamp(normalizeForIndex(maxDate)),
resourceID=self._resourceID))]
)
@@ -3674,8 +3675,8 @@
# Only needed if indexing being changed
if instanceIndexingRequired:
- values[co.RECURRANCE_MIN] = pyCalendarTodatetime(normalizeForIndex(recurrenceLowerLimit)) if recurrenceLowerLimit else None
- values[co.RECURRANCE_MAX] = pyCalendarTodatetime(normalizeForIndex(recurrenceLimit)) if recurrenceLimit else None
+ values[co.RECURRANCE_MIN] = pyCalendarToSQLTimestamp(normalizeForIndex(recurrenceLowerLimit)) if recurrenceLowerLimit else None
+ values[co.RECURRANCE_MAX] = pyCalendarToSQLTimestamp(normalizeForIndex(recurrenceLimit)) if recurrenceLimit else None
if inserting:
self._resourceID, self._created, self._modified = (
@@ -3684,15 +3685,17 @@
Return=(co.RESOURCE_ID, co.CREATED, co.MODIFIED)
).on(txn)
)[0]
+ self._created = parseSQLTimestamp(self._created)
+ self._modified = parseSQLTimestamp(self._modified)
else:
values[co.MODIFIED] = utcNowSQL
- self._modified = (
+ self._modified = parseSQLTimestamp((
yield Update(
values,
Where=co.RESOURCE_ID == self._resourceID,
Return=co.MODIFIED,
).on(txn)
- )[0][0]
+ )[0][0])
# Need to wipe the existing time-range for this and rebuild if required
if instanceIndexingRequired:
@@ -3703,8 +3706,8 @@
else:
# Keep MODIFIED the same when doing an index-only update
values = {
- co.RECURRANCE_MIN : pyCalendarTodatetime(normalizeForIndex(recurrenceLowerLimit)) if recurrenceLowerLimit else None,
- co.RECURRANCE_MAX : pyCalendarTodatetime(normalizeForIndex(recurrenceLimit)) if recurrenceLimit else None,
+ co.RECURRANCE_MIN : pyCalendarToSQLTimestamp(normalizeForIndex(recurrenceLowerLimit)) if recurrenceLowerLimit else None,
+ co.RECURRANCE_MAX : pyCalendarToSQLTimestamp(normalizeForIndex(recurrenceLimit)) if recurrenceLimit else None,
co.MODIFIED : self._modified,
}
@@ -3788,8 +3791,8 @@
tr.CALENDAR_RESOURCE_ID : self._calendar._resourceID,
tr.CALENDAR_OBJECT_RESOURCE_ID : self._resourceID,
tr.FLOATING : floating,
- tr.START_DATE : pyCalendarTodatetime(start),
- tr.END_DATE : pyCalendarTodatetime(end),
+ tr.START_DATE : pyCalendarToSQLTimestamp(start),
+ tr.END_DATE : pyCalendarToSQLTimestamp(end),
tr.FBTYPE : icalfbtype_to_indexfbtype.get(fbtype, icalfbtype_to_indexfbtype["FREE"]),
tr.TRANSPARENT : transp,
}, Return=tr.INSTANCE_ID).on(txn))[0][0]
@@ -3801,9 +3804,9 @@
def _adjustDateTime(dt, adjustment, add_duration):
if isinstance(adjustment, Duration):
- return pyCalendarTodatetime((dt + adjustment) if add_duration else (dt - adjustment))
+ return pyCalendarToSQLTimestamp((dt + adjustment) if add_duration else (dt - adjustment))
elif isinstance(adjustment, DateTime):
- return pyCalendarTodatetime(normalizeForIndex(adjustment))
+ return pyCalendarToSQLTimestamp(normalizeForIndex(adjustment))
else:
return None
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql_attachment.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql_attachment.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql_attachment.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -160,11 +160,6 @@
-def sqltime(value):
- return datetimeMktime(parseSQLTimestamp(value))
-
-
-
class AttachmentLink(object):
"""
A binding between an L{Attachment} and an L{CalendarObject}.
@@ -306,6 +301,8 @@
for attr, value in zip(child._rowAttributes(), attachmentData):
setattr(child, attr, value)
+ child._created = parseSQLTimestamp(child._created)
+ child._modified = parseSQLTimestamp(child._modified)
child._contentType = MimeType.fromString(child._contentType)
return child
@@ -404,6 +401,8 @@
the attributes may not match exactly and will need to be processed accordingly.
"""
result = dict([(attr[1:], getattr(self, attr, None)) for attr in self._rowAttributes()])
+ result["created"] = result["created"].isoformat(" ")
+ result["modified"] = result["modified"].isoformat(" ")
result["contentType"] = generateContentType(result["contentType"])
return result
@@ -469,9 +468,9 @@
for attr, value in zip(self._rowAttributes(), rows[0]):
setattr(self, attr, value)
+ self._created = parseSQLTimestamp(self._created)
+ self._modified = parseSQLTimestamp(self._modified)
self._contentType = MimeType.fromString(self._contentType)
- self._created = sqltime(self._created)
- self._modified = sqltime(self._modified)
returnValue(self)
@@ -636,11 +635,11 @@
def created(self):
- return self._created
+ return datetimeMktime(self._created)
def modified(self):
- return self._modified
+ return datetimeMktime(self._modified)
@@ -680,8 +679,8 @@
row_iter = iter(rows[0])
a_id = row_iter.next()
- created = sqltime(row_iter.next())
- modified = sqltime(row_iter.next())
+ created = parseSQLTimestamp(row_iter.next())
+ modified = parseSQLTimestamp(row_iter.next())
attachment = cls(txn, a_id, dropboxID, name, ownerHomeID, True)
attachment._created = created
@@ -754,7 +753,7 @@
att = self._attachmentSchema
self._created, self._modified = map(
- sqltime,
+ parseSQLTimestamp,
(yield Update(
{
att.CONTENT_TYPE : generateContentType(self._contentType),
@@ -845,8 +844,8 @@
row_iter = iter(rows[0])
a_id = row_iter.next()
- created = sqltime(row_iter.next())
- modified = sqltime(row_iter.next())
+ created = parseSQLTimestamp(row_iter.next())
+ modified = parseSQLTimestamp(row_iter.next())
attachment = cls(txn, a_id, ".", None, ownerHomeID, True)
attachment._managedID = managedID
@@ -1119,7 +1118,7 @@
self._size = size
att = self._attachmentSchema
self._created, self._modified = map(
- sqltime,
+ parseSQLTimestamp,
(yield Update(
{
att.CONTENT_TYPE : generateContentType(self._contentType),
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -512,7 +512,7 @@
""",
"20080601T000000Z", "20080602T000000Z",
"mailto:user1 at example.com",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ (('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),),
),
(
"#1.2 Simple component - transparent",
@@ -534,7 +534,7 @@
""",
"20080602T000000Z", "20080603T000000Z",
"mailto:user1 at example.com",
- (('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),),
+ (('N', "2008-06-02 12:00:00", "2008-06-02 13:00:00", 'B', 'T'),),
),
(
"#1.3 Simple component - canceled",
@@ -556,7 +556,7 @@
""",
"20080603T000000Z", "20080604T000000Z",
"mailto:user1 at example.com",
- (('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'F', 'F'),),
+ (('N', "2008-06-03 12:00:00", "2008-06-03 13:00:00", 'F', 'F'),),
),
(
"#1.4 Simple component - tentative",
@@ -578,7 +578,7 @@
""",
"20080604T000000Z", "20080605T000000Z",
"mailto:user1 at example.com",
- (('N', "2008-06-04 12:00:00+00:00", "2008-06-04 13:00:00+00:00", 'T', 'F'),),
+ (('N', "2008-06-04 12:00:00", "2008-06-04 13:00:00", 'T', 'F'),),
),
(
"#2.1 Recurring component - busy",
@@ -601,8 +601,8 @@
"20080605T000000Z", "20080607T000000Z",
"mailto:user1 at example.com",
(
- ('N', "2008-06-05 12:00:00+00:00", "2008-06-05 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-06 12:00:00+00:00", "2008-06-06 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-05 12:00:00", "2008-06-05 13:00:00", 'B', 'F'),
+ ('N', "2008-06-06 12:00:00", "2008-06-06 13:00:00", 'B', 'F'),
),
),
(
@@ -637,8 +637,8 @@
"20080607T000000Z", "20080609T000000Z",
"mailto:user1 at example.com",
(
- ('N', "2008-06-07 12:00:00+00:00", "2008-06-07 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-08 14:00:00+00:00", "2008-06-08 15:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-07 12:00:00", "2008-06-07 13:00:00", 'B', 'F'),
+ ('N', "2008-06-08 14:00:00", "2008-06-08 15:00:00", 'B', 'T'),
),
),
)
@@ -714,11 +714,11 @@
(
(
"user01",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),),
+ (('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'T'),),
),
(
"user02",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ (('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),),
),
),
),
@@ -767,15 +767,15 @@
(
(
"user01",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),),
+ (('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'T'),),
),
(
"user02",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ (('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),),
),
(
"user03",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ (('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),),
),
),
),
@@ -815,15 +815,15 @@
(
"user01",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'T'),
+ ('N', "2008-06-02 12:00:00", "2008-06-02 13:00:00", 'B', 'T'),
),
),
(
"user02",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),
+ ('N', "2008-06-02 12:00:00", "2008-06-02 13:00:00", 'B', 'F'),
),
),
),
@@ -875,22 +875,22 @@
(
"user01",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'T'),
+ ('N', "2008-06-02 12:00:00", "2008-06-02 13:00:00", 'B', 'T'),
),
),
(
"user02",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),
+ ('N', "2008-06-02 12:00:00", "2008-06-02 13:00:00", 'B', 'F'),
),
),
(
"user03",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),
+ ('N', "2008-06-02 12:00:00", "2008-06-02 13:00:00", 'B', 'F'),
),
),
),
@@ -945,17 +945,17 @@
(
"user01",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'T'),
+ ('N', "2008-06-02 13:00:00", "2008-06-02 14:00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00", "2008-06-03 13:00:00", 'B', 'T'),
),
),
(
"user02",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),
+ ('N', "2008-06-02 13:00:00", "2008-06-02 14:00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00", "2008-06-03 13:00:00", 'B', 'F'),
),
),
),
@@ -1025,25 +1025,25 @@
(
"user01",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'T'),
+ ('N', "2008-06-02 13:00:00", "2008-06-02 14:00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00", "2008-06-03 13:00:00", 'B', 'T'),
),
),
(
"user02",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),
+ ('N', "2008-06-02 13:00:00", "2008-06-02 14:00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00", "2008-06-03 13:00:00", 'B', 'T'),
),
),
(
"user03",
(
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-01 12:00:00", "2008-06-01 13:00:00", 'B', 'F'),
+ ('N', "2008-06-02 13:00:00", "2008-06-02 14:00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00", "2008-06-03 13:00:00", 'B', 'F'),
),
),
),
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -73,6 +73,7 @@
Update
from twext.enterprise.ienterprise import AlreadyFinishedError
from twext.enterprise.jobqueue import JobItem
+from twext.enterprise.util import parseSQLTimestamp
import datetime
import os
@@ -741,14 +742,14 @@
txn = calendarStore.newTransaction()
home = yield txn.homeWithUID(ECALENDARTYPE, "uid1", create=True)
cal = yield home.calendarWithName("calendar")
- cal._created = "2011-02-05 11:22:47"
- cal._modified = "2011-02-06 11:22:47"
+ cal._created = parseSQLTimestamp("2011-02-05 11:22:47")
+ cal._modified = parseSQLTimestamp("2011-02-06 11:22:47")
self.assertEqual(cal.created(), datetimeMktime(datetime.datetime(2011, 2, 5, 11, 22, 47)))
self.assertEqual(cal.modified(), datetimeMktime(datetime.datetime(2011, 2, 6, 11, 22, 47)))
obj = yield self.calendarObjectUnderTest()
- obj._created = "2011-02-07 11:22:47"
- obj._modified = "2011-02-08 11:22:47"
+ obj._created = parseSQLTimestamp("2011-02-07 11:22:47")
+ obj._modified = parseSQLTimestamp("2011-02-08 11:22:47")
self.assertEqual(obj.created(), datetimeMktime(datetime.datetime(2011, 2, 7, 11, 22, 47)))
self.assertEqual(obj.modified(), datetimeMktime(datetime.datetime(2011, 2, 8, 11, 22, 47)))
Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# #
+from twext.enterprise.util import parseSQLTimestamp
"""
@@ -1909,6 +1910,8 @@
for attr, value in zip(child._rowAttributes(), objectData):
setattr(child, attr, value)
+ child._created = parseSQLTimestamp(child._created)
+ child._modified = parseSQLTimestamp(child._modified)
yield child._loadPropertyStore(propstore)
@@ -2327,7 +2330,7 @@
if addressbook.owned() or addressbook.fullyShared():
rows = yield super(AddressBookObject, cls)._allColumnsWithParentAndNames(addressbook, names)
if addressbook.fullyShared() and addressbook._groupForSharedAddressBookName() in names:
- rows.append(addressbook._groupForSharedAddressBookRow())
+ rows += (addressbook._groupForSharedAddressBookRow(),)
else:
acceptedGroupIDs = yield addressbook.acceptedGroupIDs()
allowedObjectIDs = yield addressbook.expandGroupIDs(addressbook._txn, acceptedGroupIDs)
@@ -2622,6 +2625,8 @@
dataVersion=self._currentDataVersion,
)
)[0]
+ self._created = parseSQLTimestamp(self._created)
+ self._modified = parseSQLTimestamp(self._modified)
# delete foreign members table rows for this object
groupIDRows = yield Delete(
@@ -2653,7 +2658,7 @@
)
else:
- self._modified = (yield Update(
+ self._modified = parseSQLTimestamp((yield Update(
{
abo.VCARD_TEXT: self._objectText,
abo.MD5: self._md5,
@@ -2661,7 +2666,7 @@
abo.MODIFIED: utcNowSQL,
},
Where=abo.RESOURCE_ID == self._resourceID,
- Return=abo.MODIFIED).on(self._txn))[0][0]
+ Return=abo.MODIFIED).on(self._txn))[0][0])
if self._kind == _ABO_KIND_GROUP:
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -576,7 +576,7 @@
aboMembers = schema.ABO_MEMBERS
memberRows = yield Select([aboMembers.GROUP_ID, aboMembers.MEMBER_ID], From=aboMembers, Where=aboMembers.REMOVED == False).on(txn)
- self.assertEqual(memberRows, [])
+ self.assertEqual(list(memberRows), [])
aboForeignMembers = schema.ABO_FOREIGN_MEMBERS
foreignMemberRows = yield Select([aboForeignMembers.GROUP_ID, aboForeignMembers.MEMBER_ADDRESS], From=aboForeignMembers).on(txn)
@@ -607,7 +607,7 @@
)
foreignMemberRows = yield Select([aboForeignMembers.GROUP_ID, aboForeignMembers.MEMBER_ADDRESS], From=aboForeignMembers).on(txn)
- self.assertEqual(foreignMemberRows, [])
+ self.assertEqual(list(foreignMemberRows), [])
yield subgroupObject.remove()
memberRows = yield Select([aboMembers.GROUP_ID, aboMembers.MEMBER_ID, aboMembers.REMOVED, aboMembers.REVISION], From=aboMembers).on(txn)
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -29,6 +29,7 @@
from pycalendar.datetime import DateTime
+from twext.enterprise.dal.parseschema import splitSQLString
from twext.enterprise.dal.syntax import (
Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction,
Select, Update, Count, ALL_COLUMNS, Sum,
@@ -48,7 +49,7 @@
from twisted.python.util import FancyEqMixin
from twistedcaldav.config import config
-from twistedcaldav.dateops import datetimeMktime, pyCalendarTodatetime
+from twistedcaldav.dateops import datetimeMktime, pyCalendarToSQLTimestamp
from txdav.base.datastore.util import QueryCacher
from txdav.base.propertystore.none import PropertyStore as NonePropertyStore
@@ -64,7 +65,7 @@
from txdav.common.datastore.sql_notification import NotificationCollection
from txdav.common.datastore.sql_tables import _BIND_MODE_OWN, _BIND_STATUS_ACCEPTED, \
_HOME_STATUS_EXTERNAL, _HOME_STATUS_NORMAL, \
- _HOME_STATUS_PURGING, schema, splitSQLString, _HOME_STATUS_MIGRATING, \
+ _HOME_STATUS_PURGING, schema, _HOME_STATUS_MIGRATING, \
_HOME_STATUS_DISABLED
from txdav.common.datastore.sql_util import _SharedSyncLogic
from txdav.common.datastore.sql_sharing import SharingHomeMixIn, SharingMixIn
@@ -1058,7 +1059,7 @@
if cutoff < truncateLowerLimit:
raise ValueError("Cannot query events older than %s" % (truncateLowerLimit.getText(),))
- kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ kwds = {"CutOff": pyCalendarToSQLTimestamp(cutoff)}
return self._oldEventsBase(batchSize).on(self, **kwds)
@@ -1171,7 +1172,7 @@
Returns a deferred to a list of (calendar_home_owner_uid, quota used, total old size, total old count) tuples.
"""
- kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ kwds = {"CutOff": pyCalendarToSQLTimestamp(cutoff)}
if uuid:
kwds["uuid"] = uuid
@@ -1213,7 +1214,7 @@
# TODO: see if there is a better way to import Attachment
from txdav.caldav.datastore.sql import DropBoxAttachment
- kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ kwds = {"CutOff": pyCalendarToSQLTimestamp(cutoff)}
if uuid:
kwds["uuid"] = uuid
@@ -1258,7 +1259,7 @@
Returns a deferred to a list of (calendar_home_owner_uid, quota used, total old size, total old count) tuples.
"""
- kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ kwds = {"CutOff": pyCalendarToSQLTimestamp(cutoff)}
if uuid:
kwds["uuid"] = uuid
@@ -1301,7 +1302,7 @@
# TODO: see if there is a better way to import Attachment
from txdav.caldav.datastore.sql import ManagedAttachment
- kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ kwds = {"CutOff": pyCalendarToSQLTimestamp(cutoff)}
if uuid:
kwds["uuid"] = uuid
@@ -1758,6 +1759,8 @@
for attr, value in zip(self.metadataAttributes(), data):
setattr(self, attr, value)
+ self._created = parseSQLTimestamp(self._created)
+ self._modified = parseSQLTimestamp(self._modified)
def serialize(self):
@@ -1766,7 +1769,10 @@
and reconstituted at the other end. Note that the other end may have a different schema so
the attributes may not match exactly and will need to be processed accordingly.
"""
- return dict([(attr[1:], getattr(self, attr, None)) for attr in self.metadataAttributes()])
+ data = dict([(attr[1:], getattr(self, attr, None)) for attr in self.metadataAttributes()])
+ data["created"] = data["created"].isoformat(" ")
+ data["modified"] = data["modified"].isoformat(" ")
+ return data
def deserialize(self, mapping):
@@ -1776,6 +1782,8 @@
for attr in self.metadataAttributes():
setattr(self, attr, mapping.get(attr[1:]))
+ self._created = parseSQLTimestamp(self._created)
+ self._modified = parseSQLTimestamp(self._modified)
@classmethod
@@ -2568,11 +2576,11 @@
def created(self):
- return datetimeMktime(parseSQLTimestamp(self._created)) if self._created else None
+ return datetimeMktime(self._created) if self._created else None
def modified(self):
- return datetimeMktime(parseSQLTimestamp(self._modified)) if self._modified else None
+ return datetimeMktime(self._modified) if self._modified else None
@classmethod
@@ -2810,9 +2818,9 @@
returnValue(result)
try:
- self._modified = (
+ self._modified = parseSQLTimestamp((
yield self._txn.subtransaction(_bumpModified, retries=0, failureOK=True)
- )[0][0]
+ )[0][0])
yield self.invalidateQueryCache()
except AllRetriesFailed:
@@ -2932,6 +2940,8 @@
if metadataData:
for attr, value in zip(child.metadataAttributes(), metadataData):
setattr(child, attr, value)
+ child._created = parseSQLTimestamp(child._created)
+ child._modified = parseSQLTimestamp(child._modified)
# We have to re-adjust the property store object to account for possible shared
# collections as previously we loaded them all as if they were owned
@@ -3199,7 +3209,8 @@
resourceID = (yield cls._insertHomeChild.on(home._txn))[0][0]
# Initialize this object
- _created, _modified = (yield cls._insertHomeChildMetaData.on(home._txn, resourceID=resourceID))[0]
+ yield cls._insertHomeChildMetaData.on(home._txn, resourceID=resourceID)
+
# Bind table needs entry
yield cls._bindInsertQuery.on(
home._txn, homeID=home._resourceID, resourceID=resourceID, bindUID=bindUID,
@@ -3267,6 +3278,8 @@
data["bindData"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.bindAttributes()])
data["additionalBindData"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.additionalBindAttributes()])
data["metadataData"] = dict([(attr[1:], getattr(self, attr, None)) for attr in self.metadataAttributes()])
+ data["metadataData"]["created"] = data["metadataData"]["created"].isoformat(" ")
+ data["metadataData"]["modified"] = data["metadataData"]["modified"].isoformat(" ")
return data
@@ -3930,11 +3943,11 @@
def created(self):
- return datetimeMktime(parseSQLTimestamp(self._created)) if self._created else None
+ return datetimeMktime(self._created) if self._created else None
def modified(self):
- return datetimeMktime(parseSQLTimestamp(self._modified)) if self._modified else None
+ return datetimeMktime(self._modified) if self._modified else None
def addNotifier(self, factory_name, notifier):
@@ -4050,11 +4063,11 @@
returnValue(result)
try:
- self._modified = (
+ self._modified = parseSQLTimestamp((
yield self._txn.subtransaction(
_bumpModified, retries=0, failureOK=True
)
- )[0][0]
+ )[0][0])
queryCacher = self._txn._queryCacher
if queryCacher is not None:
@@ -4119,6 +4132,8 @@
for attr, value in zip(child._rowAttributes(), objectData):
setattr(child, attr, value)
+ child._created = parseSQLTimestamp(child._created)
+ child._modified = parseSQLTimestamp(child._modified)
yield child._loadPropertyStore(propstore)
@@ -4520,7 +4535,10 @@
and reconstituted at the other end. Note that the other end may have a different schema so
the attributes may not match exactly and will need to be processed accordingly.
"""
- return dict([(attr[1:], getattr(self, attr, None)) for attr in itertools.chain(self._rowAttributes(), self._otherSerializedAttributes())])
+ data = dict([(attr[1:], getattr(self, attr, None)) for attr in itertools.chain(self._rowAttributes(), self._otherSerializedAttributes())])
+ data["created"] = data["created"].isoformat(" ")
+ data["modified"] = data["modified"].isoformat(" ")
+ return data
@classmethod
@@ -4757,11 +4775,11 @@
def created(self):
- return datetimeMktime(parseSQLTimestamp(self._created))
+ return datetimeMktime(self._created)
def modified(self):
- return datetimeMktime(parseSQLTimestamp(self._modified))
+ return datetimeMktime(self._modified)
@classproperty
Modified: CalendarServer/trunk/txdav/common/datastore/sql_notification.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_notification.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/common/datastore/sql_notification.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -683,6 +683,8 @@
child._notificationType,
child._created,
child._modified,) = tuple(row)
+ child._created = parseSQLTimestamp(child._created)
+ child._modified = parseSQLTimestamp(child._modified)
try:
child._notificationType = json.loads(child._notificationType)
except ValueError:
@@ -733,6 +735,8 @@
self._notificationType,
self._created,
self._modified,) = tuple(rows[0])
+ self._created = parseSQLTimestamp(self._created)
+ self._modified = parseSQLTimestamp(self._modified)
try:
self._notificationType = json.loads(self._notificationType)
except ValueError:
@@ -828,7 +832,11 @@
notificationType=json.dumps(self._notificationType),
notificationData=notificationtext, md5=self._md5
)
- self._resourceID, self._created, self._modified = rows[0]
+ self._resourceID, self._created, self._modified = (
+ rows[0][0],
+ parseSQLTimestamp(rows[0][1]),
+ parseSQLTimestamp(rows[0][2]),
+ )
self._loadPropertyStore()
else:
rows = yield self._updateNotificationQuery.on(
@@ -836,7 +844,7 @@
notificationType=json.dumps(self._notificationType),
notificationData=notificationtext, md5=self._md5
)
- self._modified = rows[0][0]
+ self._modified = parseSQLTimestamp(rows[0][0])
self._notificationData = notificationdata
_notificationDataFromID = Select(
@@ -877,8 +885,8 @@
def created(self):
- return datetimeMktime(parseSQLTimestamp(self._created))
+ return datetimeMktime(self._created)
def modified(self):
- return datetimeMktime(parseSQLTimestamp(self._modified))
+ return datetimeMktime(self._modified)
Modified: CalendarServer/trunk/txdav/common/datastore/sql_tables.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_tables.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/common/datastore/sql_tables.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -23,13 +23,11 @@
from twext.enterprise.dal.syntax import SchemaSyntax, QueryGenerator
from twext.enterprise.dal.model import NO_DEFAULT
from twext.enterprise.dal.model import Sequence, ProcedureCall
+from twext.enterprise.dal.parseschema import schemaFromPath
from twext.enterprise.dal.syntax import FixedPlaceholder
from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT
from twext.enterprise.dal.syntax import Insert
from twext.enterprise.ienterprise import ORACLE_TABLE_NAME_MAX
-from twext.enterprise.dal.parseschema import schemaFromPath, significant
-from sqlparse import parse
-from re import compile
import hashlib
import itertools
@@ -436,44 +434,6 @@
out.write("-- Skipped Function {}\n".format(function.name))
-
-def splitSQLString(sqlString):
- """
- Strings which mix zero or more sql statements with zero or more pl/sql
- statements need to be split into individual sql statements for execution.
- This function was written to allow execution of pl/sql during Oracle schema
- upgrades.
- """
- aggregated = ''
- inPlSQL = None
- parsed = parse(sqlString)
- for stmt in parsed:
- while stmt.tokens and not significant(stmt.tokens[0]):
- stmt.tokens.pop(0)
- if not stmt.tokens:
- continue
- if inPlSQL is not None:
- agg = str(stmt).strip()
- if "end;".lower() in agg.lower():
- inPlSQL = None
- aggregated += agg
- rex = compile("\n +")
- aggregated = rex.sub('\n', aggregated)
- yield aggregated.strip()
- continue
- aggregated += agg
- continue
- if inPlSQL is None:
- # if 'begin'.lower() in str(stmt).split()[0].lower():
- if str(stmt).lower().strip().startswith('begin'):
- inPlSQL = True
- aggregated += str(stmt)
- continue
- else:
- continue
- yield str(stmt).rstrip().rstrip(";")
-
-
if __name__ == '__main__':
import sys
version = sys.argv[1] if len(sys.argv) == 2 else None
Modified: CalendarServer/trunk/txdav/common/datastore/test/test_sql_tables.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/test_sql_tables.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/common/datastore/test/test_sql_tables.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -31,8 +31,9 @@
from twext.enterprise.dal.syntax import SchemaSyntax
from txdav.common.datastore.sql_tables import schema, _translateSchema
-from txdav.common.datastore.sql_tables import SchemaBroken, splitSQLString
+from txdav.common.datastore.sql_tables import SchemaBroken
+from twext.enterprise.dal.parseschema import splitSQLString
from twext.enterprise.dal.test.test_parseschema import SchemaTestHelper
from textwrap import dedent
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -163,7 +163,7 @@
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
- yield startTxn.execSQL(path.getContent())
+ yield startTxn.execSQLBlock(path.getContent())
yield startTxn.commit()
@inlineCallbacks
@@ -269,7 +269,7 @@
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
- yield startTxn.execSQL(path.getContent())
+ yield startTxn.execSQLBlock(path.getContent())
yield startTxn.execSQL("update CALENDARSERVER set VALUE = '%s' where NAME = '%s';" % (oldVersion, versionKey,))
yield startTxn.commit()
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade_with_data.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade_with_data.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade_with_data.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -82,7 +82,7 @@
startTxn = self.store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
- yield startTxn.execSQL(path.getContent())
+ yield startTxn.execSQLBlock(path.getContent())
yield startTxn.commit()
self.addCleanup(self.cleanUp)
Modified: CalendarServer/trunk/txdav/who/test/test_group_attendees.py
===================================================================
--- CalendarServer/trunk/txdav/who/test/test_group_attendees.py 2015-03-09 19:43:14 UTC (rev 14531)
+++ CalendarServer/trunk/txdav/who/test/test_group_attendees.py 2015-03-09 19:47:39 UTC (rev 14532)
@@ -875,7 +875,7 @@
self.transactionUnderTest(),
resourceID=cobj._resourceID,
groupID=group.groupID,
- membershipHash=-1,
+ membershipHash="None",
)
wps = yield groupCacher.refreshGroup(self.transactionUnderTest(), "group01")
self.assertEqual(len(wps), 1)
@@ -1034,7 +1034,7 @@
self.transactionUnderTest(),
resourceID=cobj._resourceID,
groupID=group.groupID,
- membershipHash=-1,
+ membershipHash="None",
)
wps = yield groupCacher.refreshGroup(self.transactionUnderTest(), "group01")
self.assertEqual(len(wps), 1)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150309/7d113fdd/attachment-0001.html>
More information about the calendarserver-changes
mailing list