Refactored webserverhelper to imageprocessing

webserverhelper/binaryConverter.py → imageprocessing/binaryPreparer.py

@@ -63,20 +63,3 @@ def processBinary(path, sliceSize, sliceFrom = None, sliceTo = None, littleEndia
     floats = floats.reshape((1, numberOfSlices, numy, numx))
 
     return floats
-
-#setupEnv()
-
-#initial_image_number = 0
-#openImageFile(initial_image_number)
-#sliceImage = None
-#counter = 0
-
-#while sliceImage is not False:
- #   sliceImage = getNextSlice(numberOfBytesPerImage, numberOfBytesOfFile, imageFile, outputImagePath, (num_x, num_y))
-  #  imagehelper.saveSlice(sliceImage, counter, outputImagePath, 'out')
-   # counter += 1
-    #if image_bytes is False:
-       # logger.debug('binaryConverter image_bytes is False, aborting with counter: ')
-    #else:
-       # save_image(image_bytes, initial_image_number)
-

webserverhelper/spriteMaker.py → imageprocessing/sliceMapCreator.py

@@ -44,7 +44,7 @@ def __initializeSprite(imageFrom, imageTo, sliceShape, dtype):
 
     imagesPerRow, totalRows, imagesToProcess, numberOfSprites, slicesPerSprite, zDownScalingFactor = __calculateRowsAndColsOfSpriteMap(imageFrom, imageTo)
 
-    logger.debug('spriteMaker opened source image, writing all data into one array now')
+    logger.debug('sliceMapCreator opened source image, writing all data into one array now')
 
     outputImageArrays = []
     for counter in range(0, numberOfSprites):
@@ -92,10 +92,10 @@ def __calculateRowsAndColsOfSpriteMap(imageFrom, imageTo):
     maxImageCount = slicesPerSprite * numberOfSprites
 
     imagesLeft = maxImageCount - imagesToProcess
-    logger.debug('spriteMaker: imagesToProcess: %d, emptyImages in the last rows: %d ' % (imagesToProcess, imagesLeft))
+    logger.debug('sliceMapCreator: imagesToProcess: %d, emptyImages in the last rows: %d ' % (imagesToProcess, imagesLeft))
 
     totalRows = int(imagesPerRow)
-    logger.debug('spriteMaker imagesPerRow: %d, totalRows: %d' % (imagesPerRow, totalRows))
+    logger.debug('sliceMapCreator imagesPerRow: %d, totalRows: %d' % (imagesPerRow, totalRows))
 
     return (imagesPerRow, totalRows, imagesToProcess, numberOfSprites, slicesPerSprite, zDownScalingFactor)
 
@@ -129,7 +129,7 @@ def processFrames(frames, frameFrom, frameTo, imageFrom, imageTo, imgFormat, des
         newFrames = np.empty((frames.shape[0], frames.shape[1], newImageShape[0], newImageShape[1]), dtype=frames.dtype)
         frameCounter = 0
 
-        logger.debug('spriteMaker resizing slices to (%d, %d)' % newImageShape)
+        logger.debug('sliceMapCreator resizing slices to (%d, %d)' % newImageShape)
         for frameCounter in range(0, frames.shape[0]):
             imageCounter = 0
             cpus=cpu_count()
@@ -149,17 +149,17 @@ def processFrames(frames, frameFrom, frameTo, imageFrom, imageTo, imgFormat, des
                 process = Process(target=framePreprocessor, args=(frames[frameCounter], queues[cpu], fromImage, toImage, newImageShape,))
                 processes.append(process)
 
-            logger.debug('spriteMaker before start loop')
+            logger.debug('sliceMapCreator before start loop')
             for process in processes:
                 process.start()
 
 
             resultFrames = []
-            logger.debug('spriteMaker empty queues')
+            logger.debug('sliceMapCreator empty queues')
             for queue in queues:
                 resultFrames.append(queue.get())
 
-            logger.debug('spriteMaker merging results')
+            logger.debug('sliceMapCreator merging results')
 
             for resultFrame in resultFrames:
 
@@ -188,7 +188,7 @@ def processFrames(frames, frameFrom, frameTo, imageFrom, imageTo, imgFormat, des
         del frames
         frames = newFrames
     else:
-        logger.debug('spriteMaker no need to resize images to lower shape')
+        logger.debug('sliceMapCreator no need to resize images to lower shape')
 
     if frameFrom is not None and len(frames.shape) > 3:
         frameNumber = frameFrom
@@ -206,7 +206,7 @@ def processFrames(frames, frameFrom, frameTo, imageFrom, imageTo, imgFormat, des
     return __resizeSprites(sprites, destSize, imgFormat)
 
 def processFrame(frame, frameNumber, imageFrom, imageTo, zDownScalingFactor):
-    logger.debug('spriteMaker frameNumber %d' % frameNumber)
+    logger.debug('sliceMapCreator frameNumber %d' % frameNumber)
 
     if len(frame) is 0:
       raise ValueError('could not find any frame')
@@ -232,21 +232,21 @@ def processFrame(frame, frameNumber, imageFrom, imageTo, zDownScalingFactor):
         imageCounter += (1 * zDownScalingFactor)
         if sprite.imagesInCurRow == sprite.imagesPerRow:
             if sprite.numberOfImages >= sprite.imagesToProcess:
-                logger.debug('spriteMaker copy end reached, breaking now')
+                logger.debug('sliceMapCreator copy end reached, breaking now')
                 break
 
             sprite.currentRow += 1
             sprite.imagesInCurRow = 0
 
             if sprite.numberOfImagesInSprite == sprite.slicesPerSprite:
-                logger.debug('spriteMaker resetting currentrow to 0')
+                logger.debug('sliceMapCreator resetting currentrow to 0')
                 sprite.currentRow = 0
                 sprite.numberOfImagesInSprite = 0
 
     return sprite
 
 def generateInfo(sprites, frameFrom, frameTo, imageFrom, imageTo):
-    logger.debug('spriteMaker generating and saving .js info file')
+    logger.debug('sliceMapCreator generating and saving .js info file')
 
     firstSprite = list(sprites.values())[0][0]
     infoObject = {}
@@ -280,7 +280,7 @@ def __resizeSprites(sprites, destSize, imgFormat):
     generatorSprite = copy.deepcopy(firstSprite)
     del generatorSprite.imagesAsArrays
 
-    logger.debug('spriteMaker resizeSprites, shape of input sprites is (%d, %d)' % sprites[0].imagesAsArrays[0].shape)
+    logger.debug('sliceMapCreator resizeSprites, shape of input sprites is (%d, %d)' % sprites[0].imagesAsArrays[0].shape)
 
     if destSize is not None:
         widthUpperBound = destSize
@@ -299,7 +299,7 @@ def __resizeSprites(sprites, destSize, imgFormat):
 
     for exponentToStore in range(exponent - 1, exponentMin - 1, -1):
         destSize = math.pow(2, exponentToStore)
-        logger.debug('spriteMaker resize sprites to %d' % destSize)
+        logger.debug('sliceMapCreator resize sprites to %d' % destSize)
 
         for sprite in sprites:
             buffers = []
@@ -324,8 +324,8 @@ def __resizeSprites(sprites, destSize, imgFormat):
                     except OSError as e:
                         # due to a bug in the library of webp, the webp encoder fails for high file sizes
                         # --> if this is a big 'destSize', we have to try it again with lower quality levels
-                        logger.debug('spriteMaker failed to store webp with size of %dpx and quality value of %d' % (destSize, quality))
-                        logger.debug('spriteMaker will not use it, skipping further processing of %dpx sprite' % destSize)
+                        logger.debug('sliceMapCreator failed to store webp with size of %dpx and quality value of %d' % (destSize, quality))
+                        logger.debug('sliceMapCreator will not use it, skipping further processing of %dpx sprite' % destSize)
 
                         continue
                     contenttype += 'webp'
@@ -339,7 +339,7 @@ def __resizeSprites(sprites, destSize, imgFormat):
                     contenttype += 'jpeg'
 
 
-                logger.debug('spriteMaker image with res %d has size of: %d' % (destSize, buff.tell()))
+                logger.debug('sliceMapCreator image with res %d has size of: %d' % (destSize, buff.tell()))
 
                 buff.seek(0,0)
                 buffers.append(buff)
@@ -380,7 +380,7 @@ def calculateXYDimensions(frame):
       pixelOffset[1] = math.floor((newHeight - minSize) / 2)
       newHeight = minSize
 
-    logger.debug('spriteMaker pixeloffsets %d, %d' % (pixelOffset[0], pixelOffset[1]))
+    logger.debug('sliceMapCreator pixeloffsets %d, %d' % (pixelOffset[0], pixelOffset[1]))
 
     return pixelOffset, (newWidth, newHeight)
 

webserverhelper/tiffPreparer.py → imageprocessing/tiffPreparer.py

@@ -14,8 +14,6 @@ from skimage import img_as_ubyte
 
 from tifffile import TiffFile
 
-from . import numhelper, imagehelper
-
 import subprocess
 import shutil
 
@@ -78,12 +76,10 @@ def processTiffFile(filepath):
         z, y, x = shape
         inputFile = inputFile.reshape((1, z, y, x))
 
-    #inputFile = imagehelper.normalizeImage(inputFile)
     return inputFile
 
 def imread_convert(f):
     image = imread(f, plugin='tifffile')
-    #image = imagehelper.normalizeImage(image)
 
     return img_as_ubyte(image)
 

visualization/settings.py

@@ -197,7 +197,7 @@ LOGGING = {
             'level': 'DEBUG',
             'propagate': False,
         },
-        'webserverhelper': {
+        'imageprocessing': {
             'handlers': loggerConfiguration[ENVIRONMENT]['volumes-processing'],
             'level': 'DEBUG',
             'propagate': False,

volumes/processing/service/SliceGenerator.py

@@ -12,7 +12,7 @@ from volumes.models import Volume, Texture, TextureInfo, RawFrame
 from skimage import img_as_ubyte, img_as_int
 import numpy as np
 
-from webserverhelper import tiffPreparer, spriteMaker, binaryConverter
+from imageprocessing import tiffPreparer, sliceMapCreator, binaryPreparer
 from . import MultiProcessUByte
 
 import logging
@@ -59,7 +59,7 @@ def sliceGenerator(request, volumeId):
             frames = tiffPreparer.processTiffFile(volume.path)
         elif volume.rawData:
             logger.debug('reading raw data')
-            frames = binaryConverter.processBinary(volume.path, (volume.width, volume.height), littleEndian = volume.littleEndian)
+            frames = binaryPreparer.processBinary(volume.path, (volume.width, volume.height), littleEndian = volume.littleEndian)
         else:
             raise ValueError('sorry, file currently not supported, is it a raw file?')
 
@@ -81,10 +81,10 @@ def sliceGenerator(request, volumeId):
 
     logger.debug('cropping images to be square')
     oldshape = frames.shape
-    pixelOffset, newSize = spriteMaker.calculateXYDimensions(frames[0])
+    pixelOffset, newSize = sliceMapCreator.calculateXYDimensions(frames[0])
     croppedFrames = np.empty((frames.shape[0], frames.shape[1], newSize[1], newSize[0]), dtype=frames.dtype)
     for frameNumber in range(0, frames.shape[0]):
-        croppedFrames[frameNumber] = spriteMaker.cropFrame(frames[frameNumber], pixelOffset, newSize)
+        croppedFrames[frameNumber] = sliceMapCreator.cropFrame(frames[frameNumber], pixelOffset, newSize)
 
     del frames
     logger.debug('memory freed, frames deleted, keeping squared croppedFrames')
@@ -147,9 +147,9 @@ def sliceGenerator(request, volumeId):
     volume = Volume.objects(id=volumeId)[0]
     try:
         logger.debug('processing the image now')
-        sprites = spriteMaker.processFrames(framesUByte, 0, len(framesUByte - 1), sliceFrom, sliceTo, imgFormat, size)
+        sprites = sliceMapCreator.processFrames(framesUByte, 0, len(framesUByte - 1), sliceFrom, sliceTo, imgFormat, size)
 
-        infoObject = spriteMaker.generateInfo(sprites, 0, len(framesUByte) - 1, sliceFrom, sliceTo)
+        infoObject = sliceMapCreator.generateInfo(sprites, 0, len(framesUByte) - 1, sliceFrom, sliceTo)
     except Exception as e:
         logger.error(str(e))
         Volume.objects(id=volume.id).update_one(set__generateSlicesStatus = str(e))

volumes/processing/views.py

@@ -35,8 +35,6 @@ from PIL import Image
 from skimage import img_as_ubyte
 import numpy as np
 
-from webserverhelper import tiffPreparer, spriteMaker, binaryConverter
-
 import logging
 
 import pdb

volumes/views.py

@@ -11,8 +11,6 @@ import json
 
 from mongoengine import ValidationError
 
-from webserverhelper.imagehelper import saveSlice
-
 from .models import Volume
 from .forms import VolumeForm
 

webserverhelper/imagehelper.py

@@ -1,40 +0,0 @@
-import io, os, sys, errno
-import math
-
-import numpy as np
-
-from skimage.transform import resize
-from skimage.io import imsave
-from skimage.io import Image
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def saveSlice(sliceImage, sliceNumber, outputPath, outputPrefix, smallWidth = 300, discardBigImage = False):
-    logger.debug('imagehelper: save slice')
-    if outputPrefix.endswith('_') is False:
-        outputPrefix += '_'
-
-    # preparing file names for later saving
-    sliceNamePrefix = outputPath + '/' + outputPrefix + str(sliceNumber).zfill(4)
-    sliceName = sliceNamePrefix + '.png'
-    sliceNameSmall = sliceNamePrefix + '_small.png'
-
-    if discardBigImage is False:
-        # save big image
-        imsave(sliceName, sliceImage)
-
-    aspectRatio = float(sliceImage.shape[0]) / sliceImage.shape[1]
-    smallSize = (smallWidth, math.floor(smallWidth / aspectRatio))
-
-    # resize image into an smaller image and save it
-    imageSmall = resize(sliceImage, smallSize, order=3)
-    imsave(sliceNameSmall, imageSmall)
-
-def normalizeImage(image):
-    if image.min() < -1 or image.max() > 1:
-        image = (image - image.min()) / (image.max() - image.min())
-
-    return image

webserverhelper/numhelper.py

@@ -1,13 +0,0 @@
-def isfloat(string):
-	try:
-		float(string)
-		return True
-	except ValueError:
-		return False
-
-def isint(string):
-	try:
-		int(string)
-		return True
-	except ValueError:
-		return False