Update reconstruction

dm/client.py

@@ -61,7 +61,7 @@ class Client(object):
 
     def create_object(self, investigation_id, uploader, label=None, note=None):
         url = self.url('basemetadata/investigations', investigation_id, 'digitalObjects')
-        return Object(self.post(url, label=label, uploaderId=uploader.id, note=note))
+        return Object(self, -1, root=self.post(url, label=label, uploaderId=uploader.id, note=note))
 
     def delete_object(self, obj):
         self.delete(self.url('basemetadata/digitalObjects', obj.id))

dm/models.py

@@ -8,8 +8,10 @@ def map_entity_tags(root):
 
 
 class Object(object):
-    def __init__(self, client, oid):
-        root = client.get(client.url('basemetadata/digitalObjects', oid))
+    def __init__(self, client, oid, root=None):
+        if root is None:
+            root = client.get(client.url('basemetadata/digitalObjects', oid))
+
         entities = map_entity_tags(root)
 
         self.client = client
@@ -27,7 +29,7 @@ class Object(object):
 
     def create_ingest(self, accesspoint):
         url = self.client.url('staging/ingests')
-        return Ingest(self.client.post(url, objectId=self.uuid, accessPoint=accesspoint.uuid))
+        return Ingest(self.client, -1, root=self.client.post(url, objectId=self.uuid, accessPoint=accesspoint.uuid))
 
     def create_download(self, accesspoint):
         url = self.client.url('staging/downloads')
@@ -57,8 +59,11 @@ class IngestStatus(Enum):
 
 
 class Ingest(object):
-    def __init__(self, client, oid):
-        entities = map_entity_tags(client.get(client.url('staging/ingests', oid)))
+    def __init__(self, client, oid, root=None):
+        if root is None:
+            root = client.get(client.url('staging/ingests', oid))
+
+        entities = map_entity_tags(root)
         self.client = client
         self.id = int(entities.get('id'))
         self.staging_url = entities.get('stagingUrl')
@@ -66,7 +71,7 @@ class Ingest(object):
         self.owner_uuid = entities.get('owernUuid')
 
     def __repr__(self):
-        return '<Ingest:id={} owner_uuid={} object_uuid={} url={}>'.format(self.id, self.owner_uuid, self.object_uuid, self.staging_url)
+        return '<Ingest:id={} object_uuid={} url={} status={}>'.format(self.id, self.object_uuid, self.staging_url, self.status)
 
     @property
     def status(self):
@@ -79,9 +84,7 @@ class Ingest(object):
             raise TypeError
 
         params = dict(status=value.value, errorMessage='')
-        print params
         url = self.client.url('staging/ingests', self.id)
-        print url
 
         self.client.session.put(url, data=params)
 
@@ -134,6 +137,7 @@ class Download(object):
         self.object_uuid = entities['digitalObjectUuid']
         self.owner_uuid = entities['ownerUuid']
         self.status = entities['status']
+        self.url = entities['stagingUrl']
 
     def __repr__(self):
-        return '<Download:id={} object_uuid={}>'.format(self.id, self.object_uuid)
+        return '<Download:id={} object_uuid={} url={}>'.format(self.id, self.object_uuid, self.url)

reco.py

@@ -56,23 +56,21 @@ def configure_logging():
 
 
 def reconstruct_scan(input_path, output_path):
-    filenames = os.listdir(input_path)
     log = logging.getLogger(APP_NAME)
 
-    if 'radios' in filenames:
-        params = tofu.config.TomoParams().get_defaults()
-        params.input = os.path.join(input_path, 'radios')
-        params.from_projections = True
-        params.output = os.path.join(output_path, 'data', 'slices', 'slice-%05i.tif')
-        params.number = 1698
-        params.axis = 500
-        params.angle_step = math.radians(0.105945)
+    params = tofu.config.TomoParams().get_defaults()
+    params.input = os.path.join(input_path, 'radios')
+    params.from_projections = True
+    params.output = os.path.join(output_path, 'data', 'slices', 'slice-%05i.tif')
+    params.number = 1698
+    params.axis = 500
+    params.angle_step = math.radians(0.105945)
 
-        log.info(" Reconstruct ...")
-        proc = RecoProcess(params)
-        proc.start()
-        proc.join()
-        log.info(" Finished [output={}]".format(params.output))
+    log.info(" Reconstructing ... writing to {}".format(params.output))
+    proc = RecoProcess(params)
+    proc.start()
+    proc.join()
+    log.info(" Done.")
 
 
 class Application(object):
@@ -82,51 +80,52 @@ class Application(object):
 
     def run(self):
         last_update = get_last_timestamp()
-        self.objects = {o.uuid: o for o in self.client.get_objects(limit=100)}
+        self.objects = {o.uuid: o for o in self.client.get_objects() if o.label == 'raw'}
         self.accesspoint = self.client.get_accesspoints('file://')[0]
 
-        for ingest in self.client.get_ingests():
-            self.log.info("Found ingest={}".format(ingest.id))
-            if ingest.staging_url and ingest.id == 1:
-                self.reconstruct(ingest)
+        downloads = {d.object_uuid: d for d in self.client.get_downloads()}
 
-        now = datetime.datetime.now()
-        update_timestamp(now)
+        for uuid, obj in self.objects.items():
+            self.log.info("Checking downloads for {}".format(obj))
 
-    def reconstruct(self, ingest):
-        prefix, path = ingest.staging_url.split('file:', 1)
-
-        if os.access(path, os.R_OK):
-            self.log.info("Using ingest={} [path={} status={}]".format(ingest.id, path, ingest.status))
+            if uuid in downloads:
+                download = downloads[uuid]
+                self.log.info("Using download={}".format(download))
+                self.reconstruct(obj, download)
+            else:
+                download = obj.create_download(ap)
+                self.log.warn("No download found, created {}".format(download))
 
-            if ingest.object_uuid in self.objects:
-                obj = self.objects[ingest.object_uuid]
+        now = datetime.datetime.now()
+        update_timestamp(now)
 
-                user_client, new_ingest = self.create_ingest(obj)
+    def reconstruct(self, obj, download):
+        _, path = download.url.split('file:', 1)
+        data_path = os.path.join(path, 'data')
 
-                if new_ingest:
-                    prefix, output_path = new_ingest.staging_url.split('file:', 1)
-                    reconstruct_scan(os.path.join(path, 'data'), output_path)
+        if os.path.isdir(data_path) and 'radios' in os.listdir(data_path):
+            user_client, new_ingest = self.create_ingest(obj)
 
-                    new_ingest.status = dm.models.IngestStatus.PRE_INGEST_FINISHED
-            else:
-                self.log.warn("Object {} not found for {}".format(ingest.object_uuid, ingest.staging_url))
+            if new_ingest:
+                _, output_path = new_ingest.staging_url.split('file:', 1)
+                reconstruct_scan(data_path, output_path)
+                new_ingest.status = dm.models.IngestStatus.PRE_INGEST_FINISHED
         else:
-            self.log.error("Cannot access {}".format(path))
+            self.log.error("No raw data found in {}".format(data_path))
 
     def create_ingest(self, obj):
         try:
             user = self.client.get_user(obj.uploader)
             user_client = dm.Client(token=user.credentials.key, token_secret=user.credentials.secret)
 
-            new_obj = user_client.create_object(obj.investigation_id, user, label='foo2')
-            self.log.info(" Created new object={} [uuid={}]".format(new_obj.id, new_obj.uuid))
+            new_obj = user_client.create_object(obj.investigation_id, user, label='reco')
+            self.log.info(" Created new object={}".format(new_obj))
 
             new_ingest = new_obj.create_ingest(self.accesspoint)
 
             # get the ingest again to acccess the staging url ...
-            new_ingest = user_client.get_ingest(new_ingest.id)
-            self.log.info(" Created new ingest={} [url={}]".format(new_ingest.id, new_ingest.staging_url))
+            new_ingest = dm.models.Ingest(user_client, new_ingest.id)
+            self.log.info(" Created new ingest={}".format(new_ingest))
             return (user_client, new_ingest)
         except dm.exceptions.NotFoundError:
             self.log.warn(" No credentials found for user={}".format(obj.uploader))