Custom Data Backend
SlashDB enables its users to build and plugin custom data backends using the provided interfaces that the plugin must implement.
Python Interfaces
Any plugin implementing these interfaces will be automatically able to produce output data in formats supported by SlashDB (currently: JSON, XML, XSD, CSV and TXT).
IDBConfig
import abc
class ConnectStatus:
"""Class attribute which indicate database connection state while reflecting."""
ON = "Connected"
OFF = "Disconnected"
CONNECTING = "Connecting"
FAILED = "Failed"
class IDBConfig(object):
"""Abstract class for SlashDB configuration objects."""
__metaclass__ = abc.ABCMeta
_log_type = 'database'
_keys = [
'db_id', 'db_type', 'db_encoding', 'desc', 'autoload', 'offline',
'creator', 'owners', 'read', 'write', 'execute',
'connection', 'db_schema', 'autoload_user', 'alternate_key',
'excluded_columns', 'foreign_keys', 'connect_status'
]
def __init__(self, db_id, db_type, db_encoding, desc, autoload, offline,
creator, owners, read, write, execute,
connection, db_schema, autoload_user, alternate_key, excluded_columns, foreign_keys,
connect_status=None):
"""Sets up DBConfig object.
Validation can be skipped with validate=False if you're sure that attribute values are correct.
:param db_id: model, database id
:param db_type: type like sqlite, mssql etc.
:param connection: connection string without db_type
:param db_encoding: database charset encoding
:param desc: more detail description of the database
:param autoload_user: database user and password
:param autoload: if True then SlashDB will try to reflect database
:param offline: if True then SlashDB won't connect to database on start
:param creator: id of user that added this database config
:param owners: users with full privileges to this database config
:param read: users allowed to read this database config
:param write: users allowed to read and change this database config
:param execute: users allowed to connect/disconnect this database on demand
:param db_schema: database schema
:param alternate_key: list of columns to use as composite primary key (to be used with autoload=True)
:param excluded_columns: list of columns to exclude from table (to be used with autoload=True)
:param connect_status: state offline/connecting/connected/failed
:param foreign_keys: a dict of user defined relations
"""
self.db_id = db_id
self.db_type = db_type
self.connection = connection
self.db_encoding = db_encoding
self.desc = desc
self.autoload_user = autoload_user
self.autoload = autoload
self.offline = offline
self.creator = creator
self.owners = owners
self.read = read
self.write = write
self.execute = execute
self.db_schema = db_schema
self.alternate_key = alternate_key
self.excluded_columns = excluded_columns
self.foreign_keys = foreign_keys if foreign_keys else {}
self.connect_status = connect_status or ConnectStatus.OFF
self.errors = []
self.warnings = []
def __unicode__(self):
return u"<{}>".format(type(self).__name__)
def __repr__(self):
return "<{}>".format(type(self).__name__)
@property
def simple_dict(self):
return {key: getattr(self, key) for key in self._keys}
@property
def is_connected(self):
return self.connect_status == ConnectStatus.ON
def _extract_users(self, users_field, field_name):
"""Converts context of users_fields into list of strings.
:param users_field: contains names of users
:type users_field: string, set or list
:param field_name: field of database config that represent user group or permissions like owners.
Used in case of field owners to add creator of the database definition to the list of owners.
:type field_name: str
:returns: list of user names
:rtype: list of str
"""
u_list, is_invalid_type = parse_as_list(users_field)
if is_invalid_type:
self.errors.append('Field {} must contain comma separated user names.'.format(field_name))
return u_list
def _validate_db_id(self):
if not isinstance(self.db_id, basestring) or not self.db_id.strip():
self.errors.append('DB config id cannot be empty.')
elif not regex_db_id_name.match(self.db_id):
self.errors.append(
'Database ID may only contain letters, numbers and symbols specified in brackets [.-_@].'
)
def _validate_db_encoding(self):
if not isinstance(self.db_encoding, basestring) or not self.db_encoding.strip():
self.errors.append('Database string encoding cannot be empty.')
def _validate_autoload(self):
if isinstance(self.autoload, bool):
# keep boolean value
pass
elif str(self.autoload).strip().lower() == 'true':
self.autoload = True
elif str(self.autoload).strip().lower() == 'false':
self.autoload = False
else:
self.errors.append('Option automatically reflect DB expects True or False, got {}.'.format(self.autoload))
def _validate_offline(self):
if isinstance(self.offline, bool):
# keep boolean value
pass
elif str(self.offline).strip().lower() == 'true':
self.offline = True
elif str(self.offline).strip().lower() == 'false':
self.offline = False
else:
self.errors.append('Option offline on start expects True of False, got {}.'.format(self.offline))
@abc.abstractmethod
def validate(self, overridden_connection_string=False):
"""Validates config data."""
self.errors = []
self.warnings = []
return self.errors, self.warnings
IModel
from UserDict import DictMixin
class AttrType:
"""Class attribute which identifies types of object attributes.
It's needed by renderer to distinguish between columns,
many-to-one relation and one-to-many relation.
Used by IModel.get_attr_type method.
"""
COLUMN = 1
TOONE = 2
TOMANY = 3
class IModel(DictMixin, object):
"""Basic resource/model interface.
Reflected models are expected to implement these methods for proper rendering.
Some methods are required by DictMixin to make this object behave like a dictionary.
"""
def __getitem__(self, param):
raise NotImplementedError
def __setitem__(self, key, value):
raise NotImplementedError
def keys(self):
raise NotImplementedError
def iteritems(self):
raise NotImplementedError
def iterkeys(self):
raise NotImplementedError
def get_href(self, field_name=None, slash_escaping_fun=None):
"""Returns href without domain to resource or field or relation of the resource."""
raise NotImplementedError
def get_href_segments(self, field_name=None, slash_escaping_fun=None):
"""Returns href without domain to resource or field or relation of the resource as list of url segments."""
raise NotImplementedError
@classmethod
def get_arranged_props(cls):
"""List of strings with names of attributes/keys in preferred order (columns and relations)."""
raise NotImplementedError
@classmethod
def get_column_names(cls):
"""List of string with names of attributes/keys that refer to columns."""
raise NotImplementedError
@classmethod
def get_relation_names(cls):
"""List of strings with names of attributes/keys that refer to relationships in object."""
raise NotImplementedError
@classmethod
def get_related_class(cls, relation_name):
"""Returns a related class by given name of the relation (object attribute)."""
raise NotImplementedError
@classmethod
def get_python_type(cls, column_name):
"""Returns a correct python type for column data."""
raise NotImplementedError
@classmethod
def get_python_value(cls, column_name, text_value):
"""Converts text to proper python type value."""
raise NotImplementedError
@classmethod
def get_attr_type(cls, attr_name):
"""Returns proper slashdb.models.AttrType for queried attribute/key."""
raise NotImplementedError
def __repr__(self):
return '<{}>'.format(type(self).__name__)
IModuleMaker
class IModuleMaker(object):
"""Database model interface."""
def __init__(self, db_id, db_config, ini_settings):
"""
:param db_id - a string representing the data sources unique identifier
:param db_config - a dict-like object, representing the data sources configuration
:param ini_settings - a dictionary representing the parsed configuration *.ini file
"""
self.db_id = db_id
self.db_config = db_config
self.ini_settings = ini_settings
def create(self):
"""Returns a new python module with classes that have implemented IModel interface.
Basically
imp.new_module(self.db_id)
new_class = type(str(class_name), (ModelClass,), {'_field_names': field_names})
setattr(new_module, new_class.__name__, new_class)
return new_module
In above short example
class_name - name of a new class
ModelClass - class template that implements IModel from plugin package
{'_field_names': field_names} - class attributes of new class
"""
raise NotImplementedError
IDataProxy
class IDataProxy(object):
"""Data proxy interface.
Objects of that class implement functionality of
getting, streaming, updating, creating and deleting source data.
Attributes:
db_types (list): list of strings representing database types supported by this data proxy.
This also works as an identifier in license.
is_single_resource (bool): True if present output as single resource
For instance when PK filtering is applied.
resource_context: slashdb.datastructures.ResourceMeta object with details for the renderers
ResourceMeta(
resource_class=TheClass, # class of IModel
scalars=['field_a', 'firld_b'] # list of strings with names of fields for vector or array
)
"""
db_types = None
is_single_resource = None
resource_context = None
def __init__(self, context, request):
"""
Args:
context: instance of slashdb Context object.
Context provides
context.url_segments - an interable of information caring parts of the url i.e. ('SomeDB', 'MyTable', 'MyColumn')
context.db_id - database unique ID
context.db_config - a dict-like object, representing the data sources configuration
context.query_id - it's None if used DataDiscovery, way check if SQL Pass-thru
context.query_config - it's None if used DataDiscovery
context.authenticated_user_id - ID of the logged-in user
context.authenticated_user_config - a dict-like object, representing the users configuration
context.settings - settings prepared on start. Consider as deprecated.
This should not be used as trying to remove it from application.
context.format - requested output format based on extension and mime type.
This should be bound to available renderers formats.
Right now this is used as key to pick renderer or parser from
slashdb.formats.RENDERES or slashdb.formats.PARSERS.
request (pyramid.request.Request): pyramid request which is source of URL params, and ini settings
"""
self.context = context
self.request = request
def get(self, stream, flatten):
"""Method called by renderers to acquire data.
Streaming data is optional allows to query data in chunks and render
sequentially by renderer. That would reduce memory usage.
Args:
stream (bool): if True an iterator will be returned instead of list
flatten (bool): if True the returned data will be flattened
"""
raise NotImplementedError
def create(self, data):
"""Method that will be called to save new data into the resource.
It is called by slashdb.data_discovery_view.post_view. Together with information from context
it should be sufficient to commit new data to the resource.
Args:
data (dict): with data for new resource
"""
raise NotImplementedError
def update(self, update_data):
"""Method that will be called to update data in the resource.
It is called by slashdb.data_discovery_view.put_view. Together with information from context
it should be sufficient to commit new data to database.
Args:
update_data (dict): with data for new resource
"""
raise NotImplementedError
def delete(self):
"""Method that will be called to delete data from the resouce."""
raise NotImplementedError
def close(self):
"""Method to be called when we are done with using this resource.
Useful when there's a need to do nice cleanup on the db session or file.
Pyramid calls app_iter.close() which should call data_proxy.close().
"""
def response_postprocessing(self, response):
"""Method for modifying freshly created response.
Allows data proxy to make some changes in response like add a header.
Args:
response (pyramid.response.Response): created response object
with defined body or app_iter, content type and charset
Returns:
None
"""
pass
Example implementation of a storage plugin
For our example, we'll use a simple data structure (list of dictionaries) i.e.:
[
{'id': 1, 'name': 'John', 'age': 42},
{'id': 2, 'name': 'Adam', 'age': 36},
{'id': 2, 'name': 'Thomas', 'age': 22}
]
as the backend data storage, and the plugin itself implements access methods, employing the build-in python marshal module.
All the code, and example data can be found here.