volume-visualization/volumes/processing/service/SliceGenerator.py

import re
import time
import math
import sys

from threading import Thread
from multiprocessing import Process, cpu_count, Queue

from django.conf import settings

from volumes.models import Volume, Texture, TextureInfo, RawFrame
from volumes.processing.models import AutomaticProcessing

from skimage import img_as_ubyte, img_as_int
import numpy as np

from imageprocessing import tiffPreparer, sliceMapCreator, binaryPreparer
from . import MultiProcessUByte, Normalizer

import logging

import pdb

logger = logging.getLogger(__name__)

def generateListOfVolumes(volumesToProcess):
    automaticProcessing = AutomaticProcessing()
    automaticProcessing.finished = False
    automaticProcessing = automaticProcessing.save()

    for volumeId in volumesToProcess:
        AutomaticProcessing.objects(id=automaticProcessing.id).update_one(inc__processed = 1)
        Volume.objects(id=volumeId).update_one(set__generateSlicesStatus = 'running')

        volume = Volume.objects(id=volumeId).first()

        sliceGeneratorPlain({}, volume.id)

        if 'successful' in Volume.objects(id=volume.id).first().generateSlicesStatus:
            AutomaticProcessing.objects(id=automaticProcessing.id).update_one(inc__succeeded = 1)
        else:
            AutomaticProcessing.objects(id=automaticProcessing.id).update_one(inc__errored = 1)
            AutomaticProcessing.objects(id=automaticProcessing.id).update_one(push__erroredPaths = volume.path)


def sliceGenerator(request, volumeId):
    sliceGeneratorPlain(request.POST, volumeId)

def sliceGeneratorPlain(data, volumeId):
    '''
    This file generates the needed information and images to
    render a volume.
    For that, the volume has to be located in the local filesystem.

    The workflow to preprocess volume data is as follows:
    1. read volume data from filesystem
    2. make each slice in the volume to be square by cutting
       the edges
    3. if float image, normalize it, and covert it to int16
    4. save this data as rawframes to volume, to access it later
    5. convert image to ubyte for first preview slice maps
    6. generate slice maps
    7. save slice maps to volume
    8. save json information to volume

    '''

    sliceFrom = ''
    sliceTo = ''
    imgFormat = ''

    if 'sliceFrom' in data:
        sliceFrom = data['sliceFrom']
    if 'sliceTo' in data:
        sliceTo = data['sliceTo']
    if 'imgFormat' in data:
        imgFormat = data['imgFormat']

    size = None
    fileChanged = 'fileChanged' in data

    volume = Volume.objects(id=volumeId)[0]

    frames = []
    try:
        if volume.imageSequence is True:
            logger.debug('volId: %s, reading imagesequence' % (volumeId))
            frames = tiffPreparer.prepareImageSequence(volume.path)
        elif re.search(r'.tif(f|)$', volume.path):
            logger.debug('volId: %s, reading tiff file' % (volumeId))
            frames = tiffPreparer.processTiffFile(volume.path)
        elif volume.rawData:
            logger.debug('volId: %s, reading raw data' % (volumeId))
            frames = binaryPreparer.processBinary(volume.path, (volume.width, volume.height), littleEndian = volume.littleEndian)
        else:
            raise ValueError('sorry, file currently not supported, is it a raw file?')

    except Exception as e:
        logger.error('volId: %s, exception: %s)' % (volumeId, str(e)))
        volume.generateSlicesStatus = str(e)
        volume.save()
        return

    try:
        sliceFrom, sliceTo = __defineSliceRange(frames, sliceFrom, sliceTo)
    except Exception:
        logger.error('volId: %s, please provide integers or "" for "sliceFrom" and "sliceTo"' % (volumeId))
        Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = 'generation failed, please read the logs')
        return

    if len(frames.shape) is not 4:
        raise ValueError('read input file shall have dim of 4, please check the used fileparser')

    if frames.dtype == np.float32 or frames.dtype == np.float16:
        try:
            Normalizer.normalize_image(frames, volumeId)
        except Exception as e:
            logger.error('volId: %s, error while normalization: ' % (str(e)))
            Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = 'generation failed, please read the logs')
            return
    else:
        logger.debug('volId: %s, frames do NOT have to be normalized, dtype is %s' % (volumeId, str(frames.dtype)))

    logger.debug('volId: %s, cropping images to be square' % volumeId)
    oldshape = frames.shape
    pixelOffset, newSize = sliceMapCreator.calculateXYDimensions(frames[0])
    croppedFrames = np.empty((frames.shape[0], frames.shape[1], newSize[1], newSize[0]), dtype=frames.dtype)
    for frameNumber in range(0, frames.shape[0]):
        croppedFrames[frameNumber] = sliceMapCreator.cropFrame(frames[frameNumber], pixelOffset, newSize)

    del frames
    logger.debug('volId: %s, memory freed, frames deleted, keeping squared croppedFrames' % volumeId)

    if croppedFrames.dtype != np.int16 and croppedFrames.dtype != np.uint8:
        logger.debug('volId: %s, frames are not int16 and not uint8, convert them to int' % volumeId)
        croppedFrames = img_as_int(croppedFrames)
    else:
        logger.debug('volId: %s, frames are not converted because of dtype %s' % (volumeId, str(croppedFrames.dtype)))

    rawFrame = None
    # the file on disk has changed, we have to save it in the database
    if fileChanged or len(volume.rawFrames) == 0:
        if settings.REDUCE_HARDDISK_ACCESS_FOR_DEV is False:
            logger.debug('volId: %s, deleting rawframes from volume' % volumeId)
            volume.remove_frames()
        else:
            logger.debug('volId: %s, not deleting rawframes from volume --> reducing harddisk access' % volumeId)

        logger.debug('volId: %s, setting rawframes' % volumeId)

        try:
            for frameNumber in range(0, len(croppedFrames)):
                logger.debug('volId: %s, rawFrame: %d, sizeInMemory: %d' % (volumeId, frameNumber, sys.getsizeof(croppedFrames[frameNumber])))
                rawFrame = volume.add_frame(croppedFrames[frameNumber], frameNumber)
        except Exception:
            logger.error('volId: %s, please provide integers or "" for "sliceFrom" and "sliceTo"' % (volumeId))
            Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = 'generation failed, please read the logs')
            return
    else:
        rawFrame = volume.rawFrames[0]

    logger.debug('volId: %s, rawframes have been appended to volume' % volumeId)

    try:
        logger.debug('volId: %s, convert images to ubyte for generating preview textures' % volumeId)
        framesUByte = MultiProcessUByte.multiprocess_img_as_ubyte(croppedFrames)
        del croppedFrames
        logger.debug('volId: %s, memory freed, croppedFrames deleted, keeping ubyte frames' % volumeId)
    except Exception as e:
        logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
        Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = str(e))
        return

    logger.debug('volId: %s, converting ended' % volumeId)

    originalSize = rawFrame.originalSize

    # free more memory
    logger.debug('volId: %s, del volume object now from RAM, free memory' % volumeId)
    del volume

    volume = Volume.objects(id=volumeId)[0]
    try:
        logger.debug('volId: %s, processing the image now' % volumeId)
        sprites = sliceMapCreator.processFrames(framesUByte, 0, len(framesUByte - 1), sliceFrom, sliceTo, imgFormat, size)

        infoObject = sliceMapCreator.generateInfo(sprites, 0, len(framesUByte) - 1, sliceFrom, sliceTo)
    except Exception as e:
        logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
        Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = str(e))
        return

    try:
        volume.clear_textures()
    except Exception as e:
        logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
        Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = str(e))
        return

    try:
        logger.debug('volId: %s, save sprites in volume' % volumeId)
        for key in sprites:
            volume.add_slice_maps(int(key), sprites[key], originalSize, infoObject)
    except Exception as e:
        logger.error('volId: %s, exception: %s' % (volumeId, str(e)))
        Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = str(e))
        return

    Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = 'generation successful')
    logger.debug('volId: %s, generation successful' % volumeId)


'''
    This method returns the range of slices, that
    shall be generated
'''
def __defineSliceRange(frames, sliceFrom, sliceTo):
    if sliceFrom != '':
        sliceFrom = int(sliceFrom)
    else:
        sliceFrom = 0

    if sliceTo != '':
        sliceTo = int(sliceTo)
    else:
        dim = len(frames.shape)
        # accessing shape in z dim
        sliceTo = frames.shape[dim - 3] - 1

    return (sliceFrom, sliceTo)