[CalendarServer-changes] [6761] CalendarServer/branches/users/glyph/dal/txdav/base/datastore
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jan 19 12:56:38 PST 2011
Revision: 6761
http://trac.macosforge.org/projects/calendarserver/changeset/6761
Author: glyph at apple.com
Date: 2011-01-19 12:56:38 -0800 (Wed, 19 Jan 2011)
Log Message:
-----------
3 basic layers for SQL representation; model, parser, python convenience syntax.
Added Paths:
-----------
CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlmodel.py
CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlparser.py
CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlsyntax.py
Added: CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlmodel.py
===================================================================
--- CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlmodel.py (rev 0)
+++ CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlmodel.py 2011-01-19 20:56:38 UTC (rev 6761)
@@ -0,0 +1,166 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+class SQLType(object):
+
+ def __init__(self, name, length):
+ self.name = name
+ self.length = length
+
+
+ def __eq__(self, other):
+ if not isinstance(other, SQLType):
+ return NotImplemented
+ return (self.name, self.length) == (other.name, other.length)
+
+
+ def __ne__(self, other):
+ if not isinstance(other, SQLType):
+ return NotImplemented
+ return not self.__eq__(other)
+
+
+ def __repr__(self):
+ if self.length:
+ lendesc = '(%s)' % (self.length)
+ else:
+ lendesc = ''
+ return '<SQL Type: %r%s>' % (self.name, lendesc)
+
+
+
+class Constraint(object):
+
+ def __init__(self, type, affectsColumns, name=None):
+ self.affectsColumns = affectsColumns
+ self.name = name
+ # XXX: possibly different constraint types should have different
+ # classes?
+ self.type = type
+
+
+
+class ProcedureCall(object):
+ """
+ An invocation of a stored procedure or built-in function.
+ """
+ def __init__(self, name, args):
+ self.name = name
+ self.args = args
+
+
+class NO_DEFAULT(object):
+ """
+ Placeholder value for not having a default.
+ """
+
+
+class Column(object):
+
+ def __init__(self, table, name, type):
+ self.table = table
+ self.name = name
+ self.type = type
+ self.default = NO_DEFAULT
+ # the table object
+ self.references = None
+
+
+ def __repr__(self):
+ return '<Column (%s %r)>' % (self.name, self.type)
+
+
+ def setDefaultValue(self, value):
+ self.defaultValue = value
+
+
+ def doesReferenceName(self, name):
+ self.references = self.table.schema.tableNamed(name)
+ if self.references.primaryKey.type != self.type:
+ print 'Mismatch', self.references.primaryKey.type, self.type
+
+
+
+class Table(object):
+
+ def __init__(self, schema, name):
+ self.descriptiveComment = ''
+ self.schema = schema
+ self.name = name
+ self.columns = []
+ self.constraints = []
+ self.schemaRows = []
+ self.primaryKey = None
+ self.schema.tables.append(self)
+
+
+ def __repr__(self):
+ return '<Table %r:%r>' % (self.name, self.columns)
+
+
+ def columnNamed(self, name):
+ for column in self.columns:
+ if column.name == name:
+ return column
+ raise KeyError("no such column: %r" % (name,))
+
+
+ def addColumn(self, name, type):
+ column = Column(self, name, type)
+ self.columns.append(column)
+ return column
+
+
+ def tableConstraint(self, constraintType, columnNames):
+ affectsColumns = []
+ for name in columnNames:
+ affectsColumns.append(self.columnNamed(name))
+ self.constraints.append(Constraint(constraintType, columnNames))
+
+
+ def insertSchemaRow(self, values):
+ row = {}
+ for column, value in zip(self.columns, values):
+ row[column] = value
+ self.schemaRows.append(row)
+
+
+ def addComment(self, comment):
+ self.descriptiveComment = comment
+
+
+
+class Schema(object):
+ """
+ A schema containing tables, indexes, sequences.
+ """
+
+ def __init__(self, filename='<string>'):
+ self.filename = filename
+ self.tables = []
+
+
+ def __repr__(self):
+ return '<Schema %r>' % (self.filename,)
+
+
+ def tableNamed(self, name):
+ for table in self.tables:
+ if table.name == name:
+ return table
+ raise KeyError(name)
+
Added: CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlparser.py
===================================================================
--- CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlparser.py (rev 0)
+++ CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlparser.py 2011-01-19 20:56:38 UTC (rev 6761)
@@ -0,0 +1,287 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from itertools import chain
+
+from sqlparse import parse, keywords
+from sqlparse.tokens import Keyword, Punctuation, Number, String, Name
+from sqlparse.sql import (Comment, Identifier, Parenthesis, IdentifierList,
+ Function)
+
+from txdav.base.datastore.sqlmodel import Schema, Table, SQLType, ProcedureCall
+
+def _fixKeywords():
+ # In Postgres, 'SEQUENCE' is a keyword, and it behaves like one.
+ keywords.KEYWORDS['SEQUENCE'] = Keyword
+ # We use these as column names, and we probably shouldn't, since they _are_
+ # SQL keywords. But let's fool the parser for a moment...
+ for columnNameKeyword in ['ACCESS', 'SIZE']:
+ del keywords.KEYWORDS[columnNameKeyword]
+
+_fixKeywords()
+
+def tableFromCreateStatement(schema, stmt):
+ i = iterSignificant(stmt)
+ expect(i, ttype=Keyword.DDL, value='CREATE')
+ expect(i, ttype=Keyword, value='TABLE')
+ function = expect(i, cls=Function)
+ i = iterSignificant(function)
+ name = expect(i, cls=Identifier).get_name()
+ self = Table(schema, name)
+ parens = expect(i, cls=Parenthesis)
+ cp = _ColumnParser(self, iterSignificant(parens), parens)
+ cp.parse()
+ return self
+
+
+def schemaFromPath(path):
+ self = Schema(path.basename())
+ schemaData = path.getContent()
+ parsed = parse(schemaData)
+ for stmt in parsed:
+ preface = ''
+ while stmt.tokens and not significant(stmt.tokens[0]):
+ preface += str(stmt.tokens.pop(0))
+ if not stmt.tokens:
+ continue
+ if stmt.get_type() == 'CREATE':
+ createType = stmt.token_next(1, True)
+ if createType.value.upper() == u'TABLE':
+ t = tableFromCreateStatement(self, stmt)
+ t.addComment(preface)
+ elif stmt.get_type() == 'INSERT':
+ insertTokens = iterSignificant(stmt)
+ expect(insertTokens, ttype=Keyword.DML, value='INSERT')
+ expect(insertTokens, ttype=Keyword, value='INTO')
+ tableName = expect(insertTokens, cls=Identifier).get_name()
+ expect(insertTokens, ttype=Keyword, value='VALUES')
+ values = expect(insertTokens, cls=Parenthesis)
+ vals = iterSignificant(values)
+ expect(vals, ttype=Punctuation, value='(')
+ valuelist = expect(vals, cls=IdentifierList)
+ expect(vals, ttype=Punctuation, value=')')
+ rowData = []
+ for ident in valuelist.get_identifiers():
+ rowData.append(
+ {Number.Integer: int,
+ String.Single: _destringify}
+ [ident.ttype](ident.value)
+ )
+
+ self.tableNamed(tableName).insertSchemaRow(rowData)
+ else:
+ print 'unknown type:', stmt.get_type()
+ return self
+
+
+class _ColumnParser(object):
+ """
+ Stateful parser for the things between commas.
+ """
+
+ def __init__(self, table, parenIter, parens):
+ self.parens = parens
+ self.iter = parenIter
+ self.table = table
+
+
+ def __iter__(self):
+ return self
+
+
+ def next(self):
+ result = self.iter.next()
+ if isinstance(result, IdentifierList):
+ # Expand out all identifier lists, since they seem to pop up
+ # incorrectly. We should never see one in a column list anyway.
+ # http://code.google.com/p/python-sqlparse/issues/detail?id=25
+ while result.tokens:
+ it = result.tokens.pop()
+ if significant(it):
+ self.pushback(it)
+ return self.next()
+ return result
+
+
+ def pushback(self, value):
+ self.iter = chain(iter((value,)), self.iter)
+
+
+ def parse(self):
+ expect(self.iter, ttype=Punctuation, value=u"(")
+ while self.nextColumn():
+ pass
+
+
+ def nextColumn(self):
+ maybeIdent = self.next()
+ if maybeIdent.ttype == Name:
+ return self.parseColumn(maybeIdent.value)
+ elif isinstance(maybeIdent, Identifier):
+ return self.parseColumn(maybeIdent.get_name())
+ else:
+ return self.parseConstraint(maybeIdent)
+
+
+ def parseConstraint(self, constraintType):
+ """
+ Parse a 'free' constraint, described explicitly in the table as opposed
+ to being implicitly associated with a column by being placed after it.
+ """
+ # only know about PRIMARY KEY and UNIQUE for now
+ if constraintType.match(Keyword, 'PRIMARY'):
+ expect(self, ttype=Keyword, value='KEY')
+ expect(self, cls=Parenthesis)
+ self.primaryKey = 'MULTI-VALUE-KEY'
+ elif constraintType.match(Keyword, 'UNIQUE'):
+ expect(self, cls=Parenthesis)
+ else:
+ raise ViolatedExpectation('PRIMARY or UNIQUE', constraintType)
+ return self.checkEnd(self.next())
+
+
+ def checkEnd(self, val):
+ if val.value == u",":
+ return True
+ elif val.value == u")":
+ return False
+ else:
+ raise ViolatedExpectation(", or )", val)
+
+
+ def parseColumn(self, name):
+ typeName = self.next()
+ maybeTypeArgs = self.next()
+ if isinstance(maybeTypeArgs, Parenthesis):
+ # type arguments
+ significant = iterSignificant(maybeTypeArgs)
+ expect(significant, value=u"(")
+ typeLength = int(significant.next().value)
+ else:
+ # something else
+ typeLength = None
+ self.pushback(maybeTypeArgs)
+ theColumn = self.table.addColumn(
+ name=name, type=SQLType(typeName.value, typeLength)
+ )
+ for val in self:
+ if val.ttype == Punctuation:
+ return self.checkEnd(val)
+ else:
+ expected = True
+ if val.match(Keyword, 'PRIMARY'):
+ expect(self, ttype=Keyword, value='KEY')
+ # XXX check to make sure there's no other primary key yet
+ self.table.primaryKey = theColumn
+ elif val.match(Keyword, 'UNIQUE'):
+ # XXX add UNIQUE constraint
+ pass
+ elif val.match(Keyword, 'NOT'):
+ # no longer necessary
+ expect(self, ttype=Keyword, value='NULL')
+ elif val.match(Keyword, 'NOT NULL'):
+ # XXX add NOT NULL constraint
+ pass
+ elif val.match(Keyword, 'DEFAULT'):
+ one = self.next()
+ two = self.next()
+ if isinstance(two, Parenthesis):
+ defaultValue = ProcedureCall(one.value, two)
+ else:
+ self.pushback(two)
+ defaultValue = one.value
+ theColumn.setDefaultValue(defaultValue)
+ elif val.match(Keyword, 'REFERENCES'):
+ target = nameOrIdentifier(self.next())
+ theColumn.doesReferenceName(target)
+ elif val.match(Keyword, 'ON'):
+ expect(self, ttype=Keyword.DML, value='DELETE')
+ expect(self, ttype=Keyword, value='CASCADE')
+ else:
+ expected = False
+ if not expected:
+ print 'UNEXPECTED TOKEN:', repr(val), theColumn
+ print self.parens
+ import pprint
+ pprint.pprint(self.parens.tokens)
+ return 0
+
+
+
+
+
+class ViolatedExpectation(Exception):
+
+ def __init__(self, expected, got):
+ self.expected = expected
+ self.got = got
+ super(ViolatedExpectation, self).__init__(
+ "Expected %r got %s" % (expected, got)
+ )
+
+def nameOrIdentifier(token):
+ if isinstance(token, Identifier):
+ return token.get_name()
+ elif token.ttype == Name:
+ return token.value
+ else:
+ raise ViolatedExpectation("identifier or name", repr(token))
+
+
+def expectSingle(nextval, ttype=None, value=None, cls=None):
+ if ttype is not None:
+ if nextval.ttype != ttype:
+ raise ViolatedExpectation(ttype, '%s:%s' % (nextval.ttype, nextval))
+ if value is not None:
+ if nextval.value.upper() != value.upper():
+ raise ViolatedExpectation(value, nextval.value)
+ if cls is not None:
+ if nextval.__class__ != cls:
+ raise ViolatedExpectation(cls, repr(nextval))
+
+
+
+def expect(iterator, **kw):
+ nextval = iterator.next()
+ expectSingle(nextval, **kw)
+ return nextval
+
+
+
+def significant(token):
+ # comment has 'None' is_whitespace() result. intentional?
+ return (not isinstance(token, Comment) and not token.is_whitespace())
+
+
+
+def iterSignificant(tokenList):
+ for token in tokenList.tokens:
+ if significant(token):
+ yield token
+
+
+
+def _destringify(strval):
+ """
+ Convert a single-quoted SQL string into its actual represented value.
+ (Assumes standards compliance, since we should be controlling all the input
+ here. The only quoting syntax respected is "''".)
+ """
+ return strval[1:-1].replace("''", "'")
+
+
+
+
+
Added: CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlsyntax.py
===================================================================
--- CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlsyntax.py (rev 0)
+++ CalendarServer/branches/users/glyph/dal/txdav/base/datastore/sqlsyntax.py 2011-01-19 20:56:38 UTC (rev 6761)
@@ -0,0 +1,82 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from txdav.base.datastore.sqlmodel import Schema, Table, Column
+
+class Syntax(object):
+ """
+ Base class for syntactic convenience.
+
+ This class will define dynamic attribute access to represent its underlying
+ model as a Python namespace.
+
+ You can access the underlying model as '.model'.
+ """
+
+ modelType = None
+
+ def __init__(self, model):
+ if not isinstance(model, self.modelType):
+ # make sure we don't get a misleading repr()
+ raise ValueError("type mismatch: %r %r", type(self), model)
+ self.model = model
+
+
+ def __repr__(self):
+ return '<Syntax for: %r>' % (self.model,)
+
+
+
+class SchemaSyntax(Syntax):
+ """
+ Syntactic convenience for L{Schema}.
+ """
+
+ modelType = Schema
+
+ def __getattr__(self, attr):
+ return TableSyntax(self.model.tableNamed(attr))
+
+ def __iter__(self):
+ for table in self.model.tables:
+ yield TableSyntax(table)
+
+
+
+class TableSyntax(Syntax):
+ """
+ Syntactic convenience for L{Table}.
+ """
+
+ modelType = Table
+
+ def __getattr__(self, attr):
+ return ColumnSyntax(self.model.columnNamed(attr))
+
+
+ def __iter__(self):
+ for column in self.model.columns:
+ yield ColumnSyntax(column)
+
+
+
+class ColumnSyntax(Syntax):
+ """
+ Syntactic convenience for L{Column}.
+ """
+
+ modelType = Column
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110119/26f0e699/attachment-0001.html>
More information about the calendarserver-changes
mailing list