rewrite sql api; kitcube module; sql_models

applications/service/controllers/sql.py

@@ -1,45 +1,110 @@
 def index():
-    db = info(info.sql_db).select()
-    views = kitcube[0].views
-    # mod = info(info.sql_module).select()
-    # sensors = info(info.sql_sensor).select()
+    db = meta_dbs()
+    return locals()
 
-    # if True:
-    if False:
+def reset():
+    if True:
+    # if False:
         info.sql_db.truncate()
         info.sql_module.truncate()
         info.sql_sensor.truncate()
+    return locals()
 
-    data = heads(heads.Data_011_EBM1_DAR.id>0).select(heads.Data_011_EBM1_DAR.ALL, orderby='id DESC', limitby=(0,1))
 
-    return locals()
+def error():
+    return dict(error=session.error)    
+
+def meta_dbs():
+    return info(info.sql_db).select()
+
+def meta_modules(db):
+    return info(info.sql_db.key==db)\
+               (info.sql_module.db_id==info.sql_db.id)\
+               .select(info.sql_module.ALL)
 
-def modules():
-    db_key = session.db_key
-    m = info((info.sql_db.key==db_key) & (info.sql_module.db_id==info.sql_db.id)).select(info.sql_module.name)
-    return dict(modules=m)
+def meta_sensors(db, module):
+    return info(info.sql_db.key==db)\
+               (info.sql_module.name==module)\
+               (info.sql_sensor.db_id==info.sql_db.id)\
+               (info.sql_sensor.module_id==info.sql_module.id)\
+               .select(info.sql_sensor.ALL)
 
 @request.restful()
 def api():
     if request.extension is 'html':
-        request.extension = 'json'
-    response.view = 'generic.'+request.extension
+        request.extension = 'xml'
     if request.env.http_origin:
         response.headers['Access-Control-Allow-Origin'] = '*'
         
-    def GET(*args, **kargs):
-        if len(args) == 0: 
-            redirect('index')
-        elif len(args) == 1:
-            session.db_key = args[0]
-            redirect('modules')
-        else:
+    def GET(*args, **kargs):        
+        try:
             db_key = args[0]
-            table_name = args[1]
+        except IndexError:
+            session.error = 'no database is specified'
+            request.extension = 'html'
+            redirect('error')
+
+        databases = meta_dbs()
+
+        if db_key not in [d.key for d in databases]:
+            session.error = 'database "%s" does not exits' % db_key
+            request.extension = 'html'
+            redirect('error')
+        else:
+            modules = meta_modules(db_key)
 
-            db = globals()[db_key]
-            table = db[table_name]
-            sensors = db().select(table.ALL, orderby='id DESC', limitby=(0,1))
-        return dict(data=sensors)
+        try:
+            module = args[1]
+        except IndexError:
+            error = 'Module is not specified'
+            # mods = {}
+            # for m in modules:
+            #     s = meta_sensors(db_key, m.name)
+            #     mods[m.name] = [ _s.name for _s in s ]
+            mods = { m.name: [ s.name for s in meta_sensors(db_key, m.name) ] for m in modules }
+            # mods = { m.name : meta_sensors(db_key, m.name) for m in modules } 
+            return dict(error=error, modules=mods)
+
+        try:
+            module = modules[int(module)]
+        except ValueError:
+            module = module
+        except KeyError:
+            session.error = 'Module index out of range: module index = %s' % module
+            session.modules = modules
+            request.extension = 'html'
+            redirect(URL('error'))
+
+        len = int(kargs.get('len', 1))
+
+        db = globals()[db_key]
+        db_connector = kitcube[db_key]
+        views = db_connector.views
+
+        if module in modules.values():
+            res = db_connector.query(module, len=len)
+        elif module in views.keys():
+            v = views[module]
+            res = db_connector.query_view(v, len=len)
+        else:
+            session.error = 'Module "%s" does not exits' % module
+            session.modules = modules
+            request.extension = 'html'
+            redirect(URL('error'))
+
+        unit = {}
+        if kargs.get('unit', None):
+            if module in modules.values():
+                unit[module] = info(info.sql_sensor.module_name==module).select(info.sql_sensor.name, info.sql_sensor.unit)
+            else:
+                for mod, s in views[module]:
+                    rows = info(info.sql_sensor.module_name==mod).select(info.sql_sensor.name, info.sql_sensor.unit)
+                    rows.exclude(lambda row: row.name not in s )    
+                    unit[mod] = rows
+
+        for m, s in unit.iteritems():
+            unit[m] = { _s['name']: _s['unit'] for _s in s }
+        
+        return dict(data=res, unit=unit)
 
     return locals()

applications/service/models/db_sql.py

@@ -1,10 +1,10 @@
 from gluon.custom_import import track_changes
 track_changes(True)
 
+from sql_models import info; info._adapter.reconnect()
 from sql_models import kitcube
-for k in kitcube:
-    k.reconnect()
-    globals()[k.key] = k.dal
 
-from sql_models import info
-info._adapter.reconnect()
+for k in kitcube.keys():
+    kitcube[k].reconnect()
+    globals()[k] = kitcube[k].dal
+

applications/service/modules/config.py

@@ -48,7 +48,8 @@ sql_databases = [
                 [ 'Data_011_EBM1_DAR.INK_G_INKW_AVG',    
                   'Data_011_EBM1_DAR.OMB_N_N_002_AVG',
                   'Data_011_EBM1_DAR.SMT_M_SOIL_002_AVG',
-                  'Data_011_EBM1_DAS1.SHF_F_SHF_002_INST' ]
+                  'Data_011_EBM1_DAS1.SHF_F_SHF_002_INST',
+                  'Data_080_RPG_L1B.L1B_AZIMUTH_ANGLE' ]
         }
         # 'tables': 'Data_\d{3}_\w+'
     }

applications/service/modules/kitcube.py

@@ -37,7 +37,7 @@ class DBConnector(object):
         self.host = host
         self.database = database
         self.uri = "%s/%s" % (self.host, self.database)
-        self.dal = DAL( self.uri, migrate=False, pool_size=1, lazy_tables=True)
+        self.dal = DAL( self.uri, migrate=False, pool_size=10, lazy_tables=True)
     def extract(self):
         self._modules = self.extract_modules()
         self._sensors = self.extract_sensors(self._modules)
@@ -49,6 +49,21 @@ class DBConnector(object):
             self.dal.define_table(mod['name'], *f)
     def reconnect(self):
         self.dal._adapter.reconnect()
+    def query(self, module, fields=None, len=1):
+        db = self.dal
+        if module not in self._modules:
+            rows = []
+        if fields is None:
+            rows = db(db[module].id>0).select(db[module].ALL, orderby='id DESC', limitby=(0,len))
+        else:
+            fields = [ db[module][f] for f in fields ]
+            rows = db(db[module].id>0).select(db[module].usec, *fields, orderby='id DESC', limitby=(0,len))
+        return {module: rows.as_list()[::-1]}
+    def query_view(self, view, len=1):
+        res = {}
+        for mod, cols in view:
+            res.update(self.query(mod, cols, len=len))
+        return res
     @property
     def module_pattern(self):
         return self._module_pattern
@@ -122,7 +137,7 @@ class KITCube(DBConnector):
             cols = self.dal.executesql('show columns from ' + mod['name'])
             for fld_name, fld_type, _, fld_key, _, _ in cols:
                 if '.' in fld_name:
-                    rname = '`%s`' % fld_name
+                    rname = fld_name
                     fld_name = fld_name.replace('.', '_')
                 else:
                     rname = None
@@ -145,17 +160,18 @@ class KITCube(DBConnector):
                 if fld_type in ['mediumblob', 'blob']:
                     continue
 
-                s = search_objects(sensor_units, module=mod['mark'], name=rname)
+                s = search_objects(sensor_units, module=mod['mark'], name=rname or fld_name)
                 try:
                     unit = s[0]['unit']
                 except:
                     unit = None
 
-                sensors.append(dict(module_mark=mod['mark'],
-                                    module_name=mod['name'],
-                                    name=fld_name, type=fld_type, 
-                                    length=fld_length, rname=rname, 
-                                    unit=unit))
+                if rname is not None:
+                    rname = '`' + rname + '`'
+
+                sensors.append(dict(name=fld_name, rname=rname,
+                                    type=fld_type, length=fld_length, 
+                                    unit=unit, module_name=mod['name']))
         return sensors
 
     def extract_units(self):

applications/service/modules/sql_models.py

@@ -6,7 +6,7 @@ from kitcube import KITCube
 from helper import process_views
 
 
-kitcube = []            
+kitcube = {}            
 for database in sql_databases:
      if database['disabled']: continue
      
@@ -24,8 +24,7 @@ for database in sql_databases:
           connector.modules = modules
 
      connector.connect()
-     kitcube.append(connector)
-
+     kitcube[db_key] = connector
 
 ### Save DB meta info into database
 info = DAL('sqlite://info.storage.sqlite', 
@@ -49,10 +48,10 @@ info.define_table('sql_sensor',
                     Field('length', 'integer'),
                     Field('rname', 'string'),
                     Field('module_name', 'string'),
-                    Field('module_mark', 'integer'),
-                    Field('module', 'reference sql_module'))
+                    Field('db_id', 'reference sql_db'),
+                    Field('module_id', 'reference sql_module'))
   
-for connector in kitcube:
+for connector in kitcube.values():
      q = (info.sql_db.db==db) & (info.sql_db.host==db_host)
      r = info(q).select().last()  
      if r is None:
@@ -63,13 +62,13 @@ for connector in kitcube:
           module_ids = {}
 
           db_id = info.sql_db.insert(key=db_key, host=db_host, 
-                                   db=db, modules=module_marks)
+                                     db=db, modules=module_marks)
           for m in modules:
                module_id = info.sql_module.insert(db_id=db_id, **m)
-               module_ids[m['mark']] = module_id
+               module_ids[m['name']] = module_id
           for s in sensors:
-               module_id = module_ids[ s['module_mark'] ]
-               info.sql_sensor.insert(module=module_id, **s)
+               module_id = module_ids[ s['module_name'] ]
+               info.sql_sensor.insert(db_id=db_id, module_id=module_id, **s)
      else:
           if r.key != db_key:
                r.update_record(key=db_key)

applications/service/routes.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
 routes_in = (
-	(r'/service/sql/(?P<controller>index|modules)', r'/service/sql/\g<controller>'),
+	(r'/service/sql/(?P<controller>index|error|reset)', r'/service/sql/\g<controller>'),
 	(r'/service/sql/$anything', r'/service/sql/api/$anything'),
 )