Skip to content

Custom Data Backend

SlashDB enables its users to build and plugin custom data backends using the provided interfaces that the plugin must implement.

Python Interfaces

Any plugin implementing these interfaces will be automatically able to produce output data in formats supported by SlashDB (currently: JSON, XML, XSD, CSV and TXT).

IDBConfig

import abc


class ConnectStatus:
    """Class attribute which indicate database connection state while reflecting."""

    ON = "Connected"
    OFF = "Disconnected"
    CONNECTING = "Connecting"
    FAILED = "Failed"


class IDBConfig(object):

    """Abstract class for SlashDB configuration objects."""

    __metaclass__ = abc.ABCMeta

    _log_type = 'database'
    _keys = [
        'db_id', 'db_type', 'db_encoding', 'desc', 'autoload', 'offline',
        'creator', 'owners', 'read', 'write', 'execute',
        'connection', 'db_schema', 'autoload_user', 'alternate_key',
        'excluded_columns', 'foreign_keys', 'connect_status'
    ]

    def __init__(self, db_id, db_type, db_encoding, desc, autoload, offline,
                 creator, owners, read, write, execute,
                 connection, db_schema, autoload_user, alternate_key, excluded_columns, foreign_keys,
                 connect_status=None):
        """Sets up DBConfig object.

        Validation can be skipped with validate=False if you're sure that attribute values are correct.

        :param db_id: model, database id
        :param db_type: type like sqlite, mssql etc.
        :param connection: connection string without db_type
        :param db_encoding: database charset encoding
        :param desc: more detail description of the database
        :param autoload_user: database user and password
        :param autoload: if True then SlashDB will try to reflect database
        :param offline: if True then SlashDB won't connect to database on start
        :param creator: id of user that added this database config
        :param owners: users with full privileges to this database config
        :param read: users allowed to read this database config
        :param write: users allowed to read and change this database config
        :param execute: users allowed to connect/disconnect this database on demand
        :param db_schema: database schema
        :param alternate_key: list of columns to use as composite primary key (to be used with autoload=True)
        :param excluded_columns: list of columns to exclude from table (to be used with autoload=True)
        :param connect_status: state offline/connecting/connected/failed
        :param foreign_keys: a dict of user defined relations

        """

        self.db_id = db_id
        self.db_type = db_type
        self.connection = connection
        self.db_encoding = db_encoding
        self.desc = desc
        self.autoload_user = autoload_user
        self.autoload = autoload
        self.offline = offline
        self.creator = creator
        self.owners = owners
        self.read = read
        self.write = write
        self.execute = execute
        self.db_schema = db_schema
        self.alternate_key = alternate_key
        self.excluded_columns = excluded_columns
        self.foreign_keys = foreign_keys if foreign_keys else {}
        self.connect_status = connect_status or ConnectStatus.OFF

        self.errors = []
        self.warnings = []

    def __unicode__(self):
        return u"<{}>".format(type(self).__name__)

    def __repr__(self):
        return "<{}>".format(type(self).__name__)

    @property
    def simple_dict(self):
        return {key: getattr(self, key) for key in self._keys}

    @property
    def is_connected(self):
        return self.connect_status == ConnectStatus.ON

    def _extract_users(self, users_field, field_name):
        """Converts context of users_fields into list of strings.

        :param users_field: contains names of users
        :type users_field: string, set or list
        :param field_name: field of database config that represent user group or permissions like owners.
        Used in case of field owners to add creator of the database definition to the list of owners.
        :type field_name: str

        :returns: list of user names
        :rtype: list of str

        """
        u_list, is_invalid_type = parse_as_list(users_field)
        if is_invalid_type:
            self.errors.append('Field {} must contain comma separated user names.'.format(field_name))

        return u_list

    def _validate_db_id(self):
        if not isinstance(self.db_id, basestring) or not self.db_id.strip():
            self.errors.append('DB config id cannot be empty.')
        elif not regex_db_id_name.match(self.db_id):
            self.errors.append(
                'Database ID may only contain letters, numbers and symbols specified in brackets [.-_@].'
            )

    def _validate_db_encoding(self):
        if not isinstance(self.db_encoding, basestring) or not self.db_encoding.strip():
            self.errors.append('Database string encoding cannot be empty.')

    def _validate_autoload(self):
        if isinstance(self.autoload, bool):
            # keep boolean value
            pass
        elif str(self.autoload).strip().lower() == 'true':
            self.autoload = True
        elif str(self.autoload).strip().lower() == 'false':
            self.autoload = False
        else:
            self.errors.append('Option automatically reflect DB expects True or False, got {}.'.format(self.autoload))

    def _validate_offline(self):
        if isinstance(self.offline, bool):
            # keep boolean value
            pass
        elif str(self.offline).strip().lower() == 'true':
            self.offline = True
        elif str(self.offline).strip().lower() == 'false':
            self.offline = False
        else:
            self.errors.append('Option offline on start expects True of False, got {}.'.format(self.offline))

    @abc.abstractmethod
    def validate(self, overridden_connection_string=False):
        """Validates config data."""
        self.errors = []
        self.warnings = []
        return self.errors, self.warnings

IModel

from UserDict import DictMixin


class AttrType:
    """Class attribute which identifies types of object attributes.

    It's needed by renderer to distinguish between columns,
    many-to-one relation and one-to-many relation.

    Used by IModel.get_attr_type method.

    """

    COLUMN = 1
    TOONE = 2
    TOMANY = 3


class IModel(DictMixin, object):

    """Basic resource/model interface.

    Reflected models are expected to implement these methods for proper rendering.
    Some methods are required by DictMixin to make this object behave like a dictionary.
    """

    def __getitem__(self, param):
        raise NotImplementedError

    def __setitem__(self, key, value):
        raise NotImplementedError

    def keys(self):
        raise NotImplementedError

    def iteritems(self):
        raise NotImplementedError

    def iterkeys(self):
        raise NotImplementedError

    def get_href(self, field_name=None, slash_escaping_fun=None):
        """Returns href without domain to resource or field or relation of the resource."""
        raise NotImplementedError

    def get_href_segments(self, field_name=None, slash_escaping_fun=None):
        """Returns href without domain to resource or field or relation of the resource as list of url segments."""
        raise NotImplementedError

    @classmethod
    def get_arranged_props(cls):
        """List of strings with names of attributes/keys in preferred order (columns and relations)."""
        raise NotImplementedError

    @classmethod
    def get_column_names(cls):
        """List of string with names of attributes/keys that refer to columns."""
        raise NotImplementedError

    @classmethod
    def get_relation_names(cls):
        """List of strings with names of attributes/keys that refer to relationships in object."""
        raise NotImplementedError

    @classmethod
    def get_related_class(cls, relation_name):
        """Returns a related class by given name of the relation (object attribute)."""
        raise NotImplementedError

    @classmethod
    def get_python_type(cls, column_name):
        """Returns a correct python type for column data."""
        raise NotImplementedError

    @classmethod
    def get_python_value(cls, column_name, text_value):
        """Converts text to proper python type value."""
        raise NotImplementedError

    @classmethod
    def get_attr_type(cls, attr_name):
        """Returns proper slashdb.models.AttrType for queried attribute/key."""
        raise NotImplementedError

    def __repr__(self):
        return '<{}>'.format(type(self).__name__)

IModuleMaker

class IModuleMaker(object):
    """Database model interface."""

    def __init__(self, db_id, db_config, ini_settings):
        """
        :param db_id - a string representing the data sources unique identifier
        :param db_config - a dict-like object, representing the data sources configuration
        :param ini_settings - a dictionary representing the parsed configuration *.ini file
        """
        self.db_id = db_id
        self.db_config = db_config
        self.ini_settings = ini_settings

    def create(self):
        """Returns a new python module with classes that have implemented IModel interface.

        Basically
        imp.new_module(self.db_id)
        new_class = type(str(class_name), (ModelClass,), {'_field_names': field_names})
        setattr(new_module, new_class.__name__, new_class)
        return new_module

        In above short example
        class_name - name of a new class
        ModelClass - class template that implements IModel from plugin package
        {'_field_names': field_names} - class attributes of new class

        """
        raise NotImplementedError

IDataProxy

class IDataProxy(object):
    """Data proxy interface.

    Objects of that class implement functionality of
    getting, streaming, updating, creating and deleting source data.

    Attributes:
        db_types (list): list of strings representing database types supported by this data proxy.
            This also works as an identifier in license.

        is_single_resource (bool): True if present output as single resource
            For instance when PK filtering is applied.

        resource_context: slashdb.datastructures.ResourceMeta object with details for the renderers
            ResourceMeta(
                resource_class=TheClass,        # class of IModel
                scalars=['field_a', 'firld_b']  # list of strings with names of fields for vector or array
            )

    """

    db_types = None
    is_single_resource = None
    resource_context = None

    def __init__(self, context, request):
        """
        Args:
            context: instance of slashdb Context object.
                Context provides
                    context.url_segments - an interable of information caring parts of the url i.e. ('SomeDB', 'MyTable', 'MyColumn')
                    context.db_id - database unique ID
                    context.db_config - a dict-like object, representing the data sources configuration
                    context.query_id  - it's None if used DataDiscovery, way check if SQL Pass-thru
                    context.query_config  - it's None if used DataDiscovery
                    context.authenticated_user_id - ID of the logged-in user 
                    context.authenticated_user_config - a dict-like object, representing the users configuration
                    context.settings  - settings prepared on start. Consider as deprecated.
                        This should not be used as trying to remove it from application.
                    context.format  - requested output format based on extension and mime type.
                        This should be bound to available renderers formats.
                        Right now this is used as key to pick renderer or parser from
                        slashdb.formats.RENDERES or slashdb.formats.PARSERS.

            request (pyramid.request.Request): pyramid request which is source of URL params, and ini settings

        """
        self.context = context
        self.request = request

    def get(self, stream, flatten):
        """Method called by renderers to acquire data.

        Streaming data is optional allows to query data in chunks and render
        sequentially by renderer. That would reduce memory usage.

        Args:
            stream (bool): if True an iterator will be returned instead of list
            flatten (bool): if True the returned data will be flattened
        """
        raise NotImplementedError

    def create(self, data):
        """Method that will be called to save new data into the resource.

        It is called by slashdb.data_discovery_view.post_view. Together with information from context
        it should be sufficient to commit new data to the resource.

        Args:
            data (dict): with data for new resource

        """
        raise NotImplementedError

    def update(self, update_data):
        """Method that will be called to update data in the resource.

        It is called by slashdb.data_discovery_view.put_view. Together with information from context
        it should be sufficient to commit new data to database.

        Args:
            update_data (dict): with data for new resource

        """
        raise NotImplementedError

    def delete(self):
        """Method that will be called to delete data from the resouce."""
        raise NotImplementedError

    def close(self):
        """Method to be called when we are done with using this resource.

        Useful when there's a need to do nice cleanup on the db session or file.
        Pyramid calls app_iter.close() which should call data_proxy.close().

        """

    def response_postprocessing(self, response):
        """Method for modifying freshly created response.

        Allows data proxy to make some changes in response like add a header.

        Args:
            response (pyramid.response.Response): created response object
                with defined body or app_iter, content type and charset


        Returns:
             None

        """
        pass

Example implementation of a storage plugin

For our example, we'll use a simple data structure (list of dictionaries) i.e.:

[
    {'id': 1, 'name': 'John', 'age': 42},
    {'id': 2, 'name': 'Adam', 'age': 36},
    {'id': 2, 'name': 'Thomas', 'age': 22}
]

as the backend data storage, and the plugin itself implements access methods, employing the build-in python marshal module.

All the code, and example data can be found here.