update sql extractions

applications/service/models/config.py

@@ -1,3 +1,5 @@
+from gluon.dal import Row
+
 adei_servers = [
 {
     'server_key': 'katrin',
@@ -16,7 +18,7 @@ adei_servers = [
 
 sql_servers = [
 {
-	'server': 'orca',
+    'server': 'orca',
     'host': 'mysql://status:status@192.168.32.181',
     'database': 'orca',
     'tables': ['runs', 'machines', 'experiment'] ,
@@ -32,6 +34,24 @@ sql_servers = [
     'selector': {
         'all': [{},{}],
         'last': [{-1},{}]
-    }
+    },
+    'disabled': True
+},
+{
+    'server': 'kitcube',
+    'host': 'mysql://cube:cube@127.0.0.1',
+    'database': 'HEADS',
+    # 'tables': 'Data_\d{3}_\w*|\w*list'
+    'tables': 'Data_\d{3}_\w+'
 }
 ]
+
+
+for i, r in enumerate(sql_servers):
+    if 'tables' not in r.keys():
+        r['tables'] = []
+    if 'field_ignore' not in r.keys():
+        r['field_ignore'] = []
+    if 'field_type_fix' not in r.keys():
+        r['field_type_fix'] = {}
+    sql_servers[i].update(r)

applications/service/models/api/sql.py → applications/service/models/db_sql.py

@@ -12,14 +12,23 @@ dal_field_type = dict(
     double='double',
     float='double',
     mediumblob='blob',
+    blob='blob',
     varchar='string',
     text='text'
 )
 
+# For later convenieces,
+# copy server info into session cache
+# retrieve sql database fields, and save into session cache
+
 session.sql_servers = sql_servers
 session.sql_fields = []
 
 for c in sql_servers:
+    if 'disabled' in c.keys():
+        if c['disabled']:
+            continue
+
     host       = c['host']
     server     = c['server']
     database   = c['database']
@@ -29,42 +38,58 @@ for c in sql_servers:
 
     #_db = 'sql_' + hashlib.md5(server).hexdigest()
     _db = 'sql_' + server
-    globals()[_db] = DAL( "%s/%s" % (host, database), migrate=False, pool_size=10)
-    
+    globals()[_db] = DAL( "%s/%s" % (host, database), migrate=False, pool_size=10, lazy_tables=True)
+
+    # match table names if a pattern is defined in configuration
+    if isinstance(tables, basestring):
+        pattern = re.compile(tables)
+        tables_in_db = globals()[_db].executesql('show tables')
+        tables = [tab[0] for tab in tables_in_db if pattern.match(tab[0])]
+
+    # extract column name and type from tables
     for tab in tables:
         t0 = time.time()
+        field_info = {}
+        dal_fields = []
+        
         # columns from mysql table
         cols = globals()[_db].executesql('show columns from %s' % tab)
 
-        # define table
-        field_info = {}
-        dal_fields = []
+        # extract fields from database and construct DAL fields
+        pattern = re.compile('(?P<fld_type>\w+)\((?P<fld_length>\d*)\)')
         for fld_name, fld_type, _, fld_key, _, _ in cols:
+            
+            if '.' in fld_name:
+                rname = '`' + fld_name + '`'
+                fld_name = fld_name.replace('.', '_')
+            else:
+                rname = None
+
             field_long_name = '%s.%s' % (tab, fld_name)
-            if field_long_name in ignore_fields:
-                continue
             if field_long_name in field_type_fix.keys():
                 fld_type = field_type_fix[field_long_name]
+
+            m = pattern.match(fld_type)
+            if m:
+                fld_type = m.group('fld_type')
+                fld_length = m.group('fld_length')
             else:
-                m = re.match('(.*)\((.*)\)', fld_type)
-                if m:
-                    fld_type = m.group(1)
-                    fld_length = int(m.group(2))
-                else:
-                    length = None
-            if fld_key == 'PRI': 
+                fld_length = None
+
+            if fld_key == 'PRI':
                 fld_type = 'id'
-            field_info[fld_name] = dict(type=fld_type)
-            if fld_type == 'mediumblob':
-                continue
-            else:
-                dal_fields.append(Field(fld_name, \
-                                        type=dal_field_type[fld_type], \
-                                        length=fld_length))
 
-        globals()[_db].define_table(tab, *dal_fields)
-        session.sql_fields.append(dict(server=server, table=tab, fields=field_info))
+            if fld_type in ['mediumblob', 'blob']:
+                continue
+            if field_long_name in ignore_fields:
+                continue
 
+            field_info[fld_name] = dict(type=fld_type)
+            
+            dal_fields.append(Field(fld_name,
+                                    type=dal_field_type[fld_type],
+                                    length=fld_length,
+                                    rname=rname))
 
-#print session.sql_servers
-#print session.sql_fields
+        globals()[_db].define_table(tab, *dal_fields)
+        session.sql_fields.append(dict(server=server, table=tab, fields=field_info))