123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230 |
- import re
- import sys
- from django.conf import settings
- from volumes.models import Volume
- from volumes.processing.models import AutomaticProcessing
- from skimage import img_as_int
- import numpy as np
- from imageprocessing import tiffPreparer, sliceMapCreator, binary_preparer
- from . import MultiProcessUByte, Normalizer
- import logging, pdb
- logger = logging.getLogger(__name__)
- def generateListOfVolumes(volumesToProcess):
- automaticProcessing = AutomaticProcessing()
- automaticProcessing.finished = False
- automaticProcessing = automaticProcessing.save()
- for volumeId in volumesToProcess:
- AutomaticProcessing.objects(id=automaticProcessing.id).update_one(inc__processed=1)
- Volume.objects(id=volumeId).update_one(set__generateSlicesStatus='running')
- volume = Volume.objects(id=volumeId).first()
- sliceGeneratorPlain({}, volume.id)
- if 'successful' in Volume.objects(id=volume.id).first().generateSlicesStatus:
- AutomaticProcessing.objects(id=automaticProcessing.id).update_one(inc__succeeded=1)
- else:
- AutomaticProcessing.objects(id=automaticProcessing.id).update_one(inc__errored=1)
- AutomaticProcessing.objects(id=automaticProcessing.id).update_one(push__erroredPaths=volume.path)
- def sliceGenerator(request, volumeId):
- sliceGeneratorPlain(request.POST, volumeId)
- def sliceGeneratorPlain(data, volumeId):
- '''
- This file generates the needed information and images to
- render a volume.
- For that, the volume has to be located in the local filesystem.
- The workflow to preprocess volume data is as follows:
- 1. read volume data from filesystem
- 2. make each slice in the volume to be square by cutting
- the edges
- 3. if float image, normalize it, and covert it to int16
- 4. save this data as rawframes to volume, to access it later
- 5. convert image to ubyte for first preview slice maps
- 6. generate slice maps
- 7. save slice maps to volume
- 8. save json information to volume
- '''
- sliceFrom = ''
- sliceTo = ''
- imgFormat = ''
- if 'sliceFrom' in data:
- sliceFrom = data['sliceFrom']
- if 'sliceTo' in data:
- sliceTo = data['sliceTo']
- if 'imgFormat' in data:
- imgFormat = data['imgFormat']
- size = None
- fileChanged = 'fileChanged' in data
- volume = Volume.objects(id=volumeId)[0]
- frames = []
- try:
- if volume.imageSequence is True:
- logger.debug('volId: %s, reading imagesequence' % (volumeId))
- frames = tiffPreparer.prepareImageSequence(volume.path)
- elif re.search(r'.tif(f|)$', volume.path):
- logger.debug('volId: %s, reading tiff file' % (volumeId))
- frames = tiffPreparer.processTiffFile(volume.path)
- elif volume.rawData:
- logger.debug('volId: %s, reading raw data' % (volumeId))
- frames = binary_preparer.process_binary(volume.path, (volume.width, volume.height), little_endian=volume.littleEndian)
- else:
- raise ValueError('sorry, file currently not supported, is it a raw file?')
- except Exception as e:
- logger.error('volId: %s, exception: %s)' % (volumeId, str(e)))
- volume.generateSlicesStatus = str(e)
- volume.save()
- return
- try:
- sliceFrom, sliceTo = __defineSliceRange(frames, sliceFrom, sliceTo)
- except Exception:
- logger.error('volId: %s, please provide integers or "" for "sliceFrom" and "sliceTo"' % (volumeId))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus='generation failed, please read the logs')
- return
- if len(frames.shape) is not 4:
- raise ValueError('read input file shall have dim of 4, please check the used fileparser')
- if frames.dtype == np.float32 or frames.dtype == np.float16:
- try:
- Normalizer.normalize_image(frames, volumeId)
- except Exception as e:
- logger.error('volId: %s, error while normalization: ' % (str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus='generation failed, please read the logs')
- return
- else:
- logger.debug('volId: %s, frames do NOT have to be normalized, dtype is %s' % (volumeId, str(frames.dtype)))
- logger.debug('volId: %s, cropping images to be square' % volumeId)
- try:
- pixelOffset, newSize = sliceMapCreator.calculateXYDimensions(frames[0])
- squared_frames = sliceMapCreator.crop_frames(frames, pixelOffset, newSize)
- except Exception as e:
- logger.error('volId: %s, error while cropping: ' % (str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus='generation failed, %s' % str(e))
- return
- del frames
- logger.debug('volId: %s, memory freed, frames deleted, keeping squared squared_frames' % volumeId)
- if squared_frames.dtype != np.int16 and squared_frames.dtype != np.uint8:
- logger.debug('volId: %s, frames are not int16 and not uint8, convert them to int' % volumeId)
- squared_frames = img_as_int(squared_frames)
- else:
- logger.debug('volId: %s, frames are not converted because of dtype %s' % (volumeId, str(squared_frames.dtype)))
- rawFrame = None
- # the file on disk has changed, we have to save it in the database
- if fileChanged or len(volume.rawFrames) == 0:
- if settings.REDUCE_HARDDISK_ACCESS_FOR_DEV is False:
- logger.debug('volId: %s, deleting rawframes from volume' % volumeId)
- volume.remove_frames()
- else:
- logger.debug('volId: %s, not deleting rawframes from volume --> reducing harddisk access' % volumeId)
- logger.debug('volId: %s, setting rawframes' % volumeId)
- try:
- for frameNumber in range(0, len(squared_frames)):
- logger.debug('volId: %s, rawFrame: %d, sizeInMemory: %d' % (volumeId, frameNumber, sys.getsizeof(squared_frames[frameNumber])))
- rawFrame = volume.add_frame(squared_frames[frameNumber], frameNumber)
- except Exception as e:
- logger.error('volId: %s, please provide integers or "" for "sliceFrom" and "sliceTo" exception was: %s' % (volumeId, str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus='generation failed, please read the logs')
- return
- else:
- rawFrame = volume.rawFrames[0]
- logger.debug('volId: %s, rawframes have been appended to volume' % volumeId)
- try:
- logger.debug('volId: %s, convert images to ubyte for generating preview textures' % volumeId)
- framesUByte = MultiProcessUByte.multiprocess_img_as_ubyte(squared_frames)
- del squared_frames
- logger.debug('volId: %s, memory freed, squared_frames deleted, keeping ubyte frames' % volumeId)
- except Exception as e:
- logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus=str(e))
- return
- logger.debug('volId: %s, converting ended' % volumeId)
- originalSize = rawFrame.originalSize
- # free more memory
- logger.debug('volId: %s, del volume object now from RAM, free memory' % volumeId)
- del volume
- volume = Volume.objects(id=volumeId)[0]
- try:
- logger.debug('volId: %s, processing the image now' % volumeId)
- sprites = sliceMapCreator.processFrames(framesUByte, 0, len(framesUByte - 1), sliceFrom, sliceTo, imgFormat, size)
- infoObject = sliceMapCreator.generateInfo(sprites, 0, len(framesUByte) - 1, sliceFrom, sliceTo)
- except Exception as e:
- logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus=str(e))
- return
- try:
- volume.clear_textures()
- except Exception as e:
- logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus=str(e))
- return
- try:
- logger.debug('volId: %s, save sprites in volume' % volumeId)
- for key in sprites:
- volume.add_slice_maps(int(key), sprites[key], originalSize, infoObject)
- except Exception as e:
- logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus=str(e))
- return
- Volume.objects(id=volume.id).update_one(set__generateSlicesStatus='generation successful')
- logger.debug('volId: %s, generation successful' % volumeId)
- def __defineSliceRange(frames, sliceFrom, sliceTo):
- '''
- This method returns the range of slices, that
- shall be generated
- '''
- if sliceFrom != '':
- sliceFrom = int(sliceFrom)
- else:
- sliceFrom = 0
- if sliceTo != '':
- sliceTo = int(sliceTo)
- else:
- dim = len(frames.shape)
- # accessing shape in z dim
- sliceTo = frames.shape[dim - 3] - 1
- return (sliceFrom, sliceTo)
|