reorgnize sql_models

applications/service/controllers/sql.py

@@ -1,5 +1,6 @@
 def index():
     db = info(info.sql_db).select()
+    views = kitcube[0].views
     # mod = info(info.sql_module).select()
     # sensors = info(info.sql_sensor).select()
 
@@ -13,8 +14,13 @@ def index():
 
     return locals()
 
+def modules():
+    db_key = session.db_key
+    m = info((info.sql_db.key==db_key) & (info.sql_module.db_id==info.sql_db.id)).select(info.sql_module.name)
+    return dict(modules=m)
+
 @request.restful()
-def data():
+def api():
     if request.extension is 'html':
         request.extension = 'json'
     response.view = 'generic.'+request.extension
@@ -25,12 +31,13 @@ def data():
         if len(args) == 0: 
             redirect('index')
         elif len(args) == 1:
-            db_name = args[0]
+            session.db_key = args[0]
+            redirect('modules')
         else:
-            db_name = args[0]
+            db_key = args[0]
             table_name = args[1]
 
-            db = globals()[db_name]
+            db = globals()[db_key]
             table = db[table_name]
             sensors = db().select(table.ALL, orderby='id DESC', limitby=(0,1))
         return dict(data=sensors)

applications/service/models/db_sql.py

@@ -1,11 +1,10 @@
 from gluon.custom_import import track_changes
 track_changes(True)
 
-from sql_models import info
 from sql_models import kitcube
-
-info._adapter.reconnect()
-
 for k in kitcube:
     k.reconnect()
-    globals()[k.key] = k.connector
+    globals()[k.key] = k.dal
+
+from sql_models import info
+info._adapter.reconnect()

applications/service/modules/config.py

@@ -42,7 +42,14 @@ sql_databases = [
         'db': 'HEADS',
         'modules': 'Data_(?P<mod_mark>\d{3})_\w+',
         'axislist': 'Axislist',
-        'sensorlist': 'Sensorlist'
+        'sensorlist': 'Sensorlist',
+        'views': {
+            'test1': 
+                [ 'Data_011_EBM1_DAR.INK_G_INKW_AVG',    
+                  'Data_011_EBM1_DAR.OMB_N_N_002_AVG',
+                  'Data_011_EBM1_DAR.SMT_M_SOIL_002_AVG',
+                  'Data_011_EBM1_DAS1.SHF_F_SHF_002_INST' ]
+        }
         # 'tables': 'Data_\d{3}_\w+'
     }
 ]

applications/service/modules/helper.py

@@ -0,0 +1,116 @@
+def query_as_dict(expression, field_type=None):
+    """Seralizes gluon.dal.Query as dictionary.
+    
+    Converts a gluon.dal.Query or gluon.dal.Expression
+    into a dictionary structure that can be pickled and
+    stored in the session.
+    
+    Args:
+        expression: gluon.dal.Query or gluon.dal.Expression
+    
+    Returns:
+        Dictionary in the form {op:op, first:expression, second:expression}
+        op: the query operation (eg, AND, EQ, GT)
+        expression: either a dictionary (that expands a gluon Table,
+            Field, Expression or Query object) or a base object such as
+            a string or list.
+    
+    For example:
+        >>>query = (db.comment.id.belongs((1,2,3))) & (db.webpage.title == 'FAQ')
+        >>>print query_as_dict(query)
+        "{'second': {'second': 'FAQ', 'first': {'table': 'webpage', 'fieldname': 'title',
+        'tablename': 'webpage'}, 'op': 'EQ'}, 'first': {'second': (1, 2, 3), 'first':
+        {'table': 'comment', 'fieldname': 'id', 'tablename': 'comment'}, 'op': 'BELONGS'},
+        'op': 'AND'}"
+    """
+    from gluon.dal import Query, Expression, Table, Field
+    if isinstance(expression, Field):
+        tablename = expression._tablename
+        return dict(tablename=expression._tablename,
+                    table = str(expression._table),
+                    fieldname = expression.name)
+    elif isinstance(expression, (Expression, Query)):
+        if not expression.second is None:
+            return dict(op=expression.op.im_func.__name__, 
+                        first=query_as_dict(expression.first), 
+                        second=query_as_dict(expression.second))
+        elif not expression.first is None:
+            if not expression.op is None:
+                return dict(op=expression.op.im_func.__name__,
+                            first=query_as_dict(expression.first),
+                            second=None) # eg '(person.title IS NULL)'
+            else:
+                return expression.first
+        elif not isinstance(expression.op, str):
+            return expression.op()
+        else:
+            return '(%s)' % expression.op
+    elif field_type:
+        return str(represent(expression,field_type))
+    elif isinstance(expression,(list,tuple)):
+        return expression
+    elif isinstance(expression, Table):
+        return dict(tablename=expression._tablename,
+                    table = str(expression))
+    elif expression==None:
+        return None
+    else:
+        return str(expression)
+
+
+def query_from_dict(db, query, out='Query'):
+    """Builds gluon.dal.Query from dictionary structure.
+        
+    Args:
+        db: gluon.dal.db object
+        query: A dictionary in the form {op:op, first:expression, second:expression}
+            as returned by query_as_dict()
+        out: Set to 'Expression' for gluon.dal.Expression 
+            rather than Query.
+    
+    Returns:
+        gluon.dal.Query or gluon.dal.Expression object
+    """
+    from gluon.dal import Expression, Query
+    if out == 'Expression':
+        out_class = Expression
+    else:
+        out_class = Query
+    if type(query) == dict:
+        if 'op' in query.keys():
+            first = query_from_dict(db, query['first'], out=out)
+            second = query_from_dict(db, query['second'], out=out)
+            op = getattr(db._adapter, query['op'])
+            return out_class(db, op, first=first, second=second)
+        elif 'fieldname' in query.keys():
+            if query['tablename'] == query['table']:
+                return db[query['tablename']][query['fieldname']]
+            else: # a table.field with alias
+                return db[query['table']].with_alias(query['tablename'])[query['fieldname']]
+        elif 'tablename' in query.keys():
+            if query['tablename'] == query['table']:
+                return db[query['tablename']]
+            elif ' AS ' in query['table']: # a table with alias
+                t = query['table'].split(' ')[0]
+                return db[t].with_alias(query['tablename'])          
+        else:
+            raise ValueError
+    else:
+        return query
+
+def process_views(views):
+    import re
+    from itertools import groupby
+    pattern = re.compile('(\w+).(\w+)')
+    keyfunc = lambda x: x[0]
+    grouped_views = {}
+    for view_name, sensors in views.iteritems():
+        slist = []
+        for sensor in sensors:
+            m = pattern.match(sensor)
+            slist.append((m.group(1), m.group(2)))
+        sgroups = []
+        for table, sgroup in groupby(slist, keyfunc):
+            sgroups.append((table, [s[1] for s in sgroup]))
+        grouped_views[view_name] = sgroups
+    return grouped_views

applications/service/modules/kitcube.py

@@ -25,59 +25,83 @@ def search_objects2(axes, sensor_name, module_mark):
             return ax['unit']
     return None
 
-
+        
 class DBConnector(object):
     def __init__(self, key, host, database):
+        self._modules = None
+        self._views = None
+        self._module_pattern = None
         self.axis_list = 'Axislist'
         self.sensor_list = 'Sensorlist'
         self.key = key
         self.host = host
         self.database = database
         self.uri = "%s/%s" % (self.host, self.database)
-        self.connector = DAL( self.uri, migrate=False, pool_size=1, lazy_tables=True)
+        self.dal = DAL( self.uri, migrate=False, pool_size=1, lazy_tables=True)
     def extract(self):
-        modules = self.extract_modules()
-        sensors = self.extract_sensors(modules)
-        return modules, sensors
-    def connect(self, modules=None, sensors=None):
-        if modules is None or sensors is None:
-            modules, sensors = extract()
-        for mod in modules:
-            s = search_objects(sensors, module_name=mod['name'])
+        self._modules = self.extract_modules()
+        self._sensors = self.extract_sensors(self._modules)
+    def connect(self):
+        self.extract()
+        for mod in self._modules:
+            s = search_objects(self._sensors, module_name=mod['name'])
             f = [ Field(k['name'], type=k['type'], length=k['length'], rname=k['rname']) for k in s ]
-            self.connector.define_table(mod['name'], *f)
+            self.dal.define_table(mod['name'], *f)
     def reconnect(self):
-        self.connector._adapter.reconnect()
-
+        self.dal._adapter.reconnect()
+    @property
+    def module_pattern(self):
+        return self._module_pattern
+    @module_pattern.setter
+    def module_pattern(self, value):
+        self._module_pattern = re.compile(value)
+    @property
+    def modules(self):
+        return self._modules
+    @modules.setter
+    def modules(self, value):
+        self._modules = value
+    @property
+    def sensors(self):
+        return self._sensors
+    @property
+    def views(self):
+        return self._views
+    @views.setter
+    def views(self, value):
+        self._views = value
+    
+    
+    
 class KITCube(DBConnector):
-    def __init__(self, key, host, database, modules, ignore_fields=[], field_type_fix={}):
+    def __init__(self, key, host, database, ignore_fields=[], field_type_fix={}):
         self.ignore_fields = ignore_fields
         self.field_type_fix = field_type_fix
         super(KITCube, self).__init__(key, host, database)
 
-        if isinstance(modules, basestring):
-            self.module_pattern = re.compile(modules)
-        else:
-            self.module_pattern = None
-
     def extract_modules(self):
-        ''' extract modules from database
+        ''' extract modules 
+            extrace from database when self._modules is not defined,
+            otherwise filter self._modules, 
             
-            @return 
+            Returns: 
                 [{'name': <name>, 'mark': <module number>}, ... ]
         '''
-        tables = self.connector.executesql('show tables')
-        if self.module_pattern is not None:
-            matches = [ (tname, self.module_pattern.match(tname)) for tname, in tables ]
+        tables = [ tname for tname, in self.dal.executesql('show tables') ]
+        if self._modules is None:
+            matches = [ (tname, self.module_pattern.match(tname)) for tname in tables ]
             modules = [ dict(name=tname, mark=int(m.group('mod_mark'))) for tname, m in matches if m ]
+        else:
+            modules = [ t for t in self._modules if t in tables ]
         return modules
 
     def extract_sensors(self, modules):
         ''' extract sensors from database
 
-            @input modules 
-                list of moudles to be extracted
-            @return:
+            Args: 
+                modules: list of moudles to be extracted
+            
+            Returns:
                 [ {
                     'name': <sensor name>,       # dots in name is coverted to underscore
                     'rname': <sensor rname>,     # original name from db, if it contains dots
@@ -95,7 +119,7 @@ class KITCube(DBConnector):
         sensors = []
 
         for mod in modules:
-            cols = self.connector.executesql('show columns from ' + mod['name'])
+            cols = self.dal.executesql('show columns from ' + mod['name'])
             for fld_name, fld_type, _, fld_key, _, _ in cols:
                 if '.' in fld_name:
                     rname = '`%s`' % fld_name
@@ -135,17 +159,17 @@ class KITCube(DBConnector):
         return sensors
 
     def extract_units(self):
-        self.connector.define_table(self.axis_list,   
+        self.dal.define_table(self.axis_list,   
                             Field('id'), Field('unit', 'string'))
-        self.connector.define_table(self.sensor_list, 
+        self.dal.define_table(self.sensor_list, 
                             Field('id'), Field('name', 'string'),
                             Field('module', 'integer'),
                             Field('axis', 'reference %s' % self.axis_list))
         
-        sensortable = self.connector[self.sensor_list]
-        axistable = self.connector[self.axis_list]
+        sensortable = self.dal[self.sensor_list]
+        axistable = self.dal[self.axis_list]
         
-        records = self.connector(sensortable.axis==axistable.id).select()
+        records = self.dal(sensortable.axis==axistable.id).select()
         sensor_units = [ dict(name=r[self.sensor_list].name, 
                               module=r[self.sensor_list].module, 
                               unit=r[self.axis_list].unit) for r in records ]

applications/service/modules/sql_models.py

@@ -1,8 +1,33 @@
+import re
+from itertools import groupby
 from gluon.dal import DAL, Field
 from config import sql_databases
 from kitcube import KITCube
+from helper import process_views
 
 
+kitcube = []            
+for database in sql_databases:
+     if database['disabled']: continue
+     
+     db_key  = database['key']
+     db_host = database['host']
+     db      = database['db']
+     modules = database['modules']
+     views   = database.get('views', None)
+
+     connector = KITCube(db_key, db_host, db)
+     connector.views = process_views(views)
+     if isinstance(modules, basestring):
+          connector.module_pattern = modules
+     else:
+          connector.modules = modules
+
+     connector.connect()
+     kitcube.append(connector)
+
+
+### Save DB meta info into database
 info = DAL('sqlite://info.storage.sqlite', 
            pool_size=10,
            check_reserved=False)
@@ -27,22 +52,13 @@ info.define_table('sql_sensor',
                     Field('module_mark', 'integer'),
                     Field('module', 'reference sql_module'))
   
-kitcube = []
-
-for database in sql_databases:
-     if database['disabled']: continue
-     db_key  = database['key']
-     db_host = database['host']
-     db      = database['db']
-     modules = database['modules']
-    
+for connector in kitcube:
      q = (info.sql_db.db==db) & (info.sql_db.host==db_host)
      r = info(q).select().last()  
-
-     connector = KITCube(db_key, db_host, db, modules)    
-    
      if r is None:
-          modules, sensors = connector.extract()
+          modules = connector.modules
+          sensors = connector.sensors
+
           module_marks = [ m['mark'] for m in modules]
           module_ids = {}
 
@@ -58,11 +74,5 @@ for database in sql_databases:
           if r.key != db_key:
                r.update_record(key=db_key)
 
-          modules = info(info.sql_module).select()
-          sensors = info(info.sql_sensor).select()
-
-     connector.connect(modules, sensors)
-     kitcube.append(connector)
-