KALYPSO2_5_GUI/strip_data.py

import numpy as np
import sys


pixels = 512*2
TAIL = [0xba98,0xfedc]*16
DEAD = [0xdea1,0xdead,0xdea2,0xdead,0xdea3,0xdead,0xdea4,0xdead,0xdea5,0xdead,0xdea6,0xdead,0xdea7,0xdead,0xdea8,0xdead]

def strip_data(filename, number_pixels, orbits, offset = None):

        ###############----------Read data from memory----------############################################
        np.set_printoptions(threshold='nan')
        # data = np.fromfile(str(filename), dtype=np.uint16)
        if 	offset is None:
        	data = np.memmap( str(filename), dtype=np.dtype('<u2'), mode='r')
        else:
            data = np.memmap( str(filename), dtype=np.dtype('<u2'), offset = (2*pixels*offset),mode='r')


        ###############----------Remove Header----------####################################################

        if (data[0] == 0x1111 and data[15] == 0xF888):
            print ('Removing header .....')
            data = data[16:]
        else:
            print ('No data header....')
            data = data

        ##############--------Search and remove for TAIL--------############################################
        offset = 0
        tailoffset = offset
        while np.any((data[tailoffset : tailoffset + len(TAIL)] != TAIL)):
            tailoffset += 8
            if tailoffset + len(TAIL) > len(data):
                print('Could not find tail')
                #sys.exit(-1)
                data = data
        print('Found tail at offset ' + str(tailoffset))

        ##############--------Search and remove for DEAD filling--------##################################
        deadcnt = 0
        deadoffset = tailoffset
        while np.any(data[deadoffset - len(DEAD) : deadoffset] == DEAD):
            deadcnt +=1
            deadoffset -= len(DEAD)

        print('Removed %d bytes of DEAD' %(deadcnt))

        ##############--------Set the pure data --------###################################################

        data = np.frombuffer((data[offset:deadoffset]), dtype = np.uint16)

        #dead = np.where(data == 0xdea1)
        #loc = np.amin(dead)
        #data = data[0:loc]

        ############-------Swap the data bytes and change from 16 bit to 14 bit---------####################

        data = data.byteswap()


        #############---------Index decoder for the jesd data for 16 input mode----------###################
        odd_inner_idx_1 = [23,22,7,6,21,20,5,4,19,18,3,2,17,16,1,0] # working one
        odd_inner_idx_2 = [8,9,24,25,10,11,26,27,12,13,28,29,14,15,30,31]    # non interleaved

        odd_index_1 = np.zeros([256], dtype=np.int64)
        odd_index_2 = np.zeros([256], dtype=np.int64)
        for i in range(16):
            for j in range(16):
                odd_index_1[i * 16 + j] = odd_inner_idx_1[i] +  j * 64

        for i in range(16):
            for j in range(16):
                odd_index_2[i * 16 + j] = odd_inner_idx_2[i] +  j * 64


        even_inner_idx_1 = [55,54,39,38,53,52,37,36,51,50,35,34,49,48,33,32] # working one
        even_inner_idx_2 = [40,41,56,57,42,43,58,59,44,45,60,61,46,47,62,63]    # non interleaved

        even_index_1 = np.zeros([256], dtype=np.int64)
        even_index_2 = np.zeros([256], dtype=np.int64)
        for i in range(16):
            for j in range(16):
                even_index_1[i * 16 + j] = even_inner_idx_1[i] +  j * 64

        for i in range(16):
            for j in range(16):
                even_index_2[i * 16 + j] = even_inner_idx_2[i] +  j * 64


        #temp = [even_index_1[i^1] for i in range(len(even_index_1))]
        #even_index_1 = temp


        #swap odd left adc pixels
        #odd_index_1_t = odd_index_1 + 1
        #odd_index_1_t[:,0::2], odd_index_1_t[:,1::2] = odd_index_1[:,1::2], odd_index_1[:,0::2]
        #odd_index_1 = odd_index_1_t

        #swap even left adc pixels
        #even_index_1_t = even_index_1 + 1
        #even_index_1_t[:,0::2], even_index_1_t[:,1::2] = even_index_1[:,1::2], even_index_1[:,0::2]
        #even_index_1 = even_index_1_t


        ################------working part---------###########################################################
        #index_interleave = np.ravel(np.column_stack((index_1,index_2)))  #interleaving the pixels, NOTE: interchange index_1 & index_2 if the pixels are swapped
        #k = 32
        #pixel_map = []
        #a = index_interleave

        #for i in range(0, len(a), k):
        #    pixel_map.extend((a[i:i + k])[::-1])
        ######################################################################################################
        l = 16
        odd_adc_2 = []
        odd_b = odd_index_2
        for i in range(0, len(odd_b), l):
            odd_adc_2.extend((odd_b[i:i + l])[::-1])

        even_adc_2 = []
        even_b = even_index_2
        for i in range(0, len(even_b), l):
            even_adc_2.extend((even_b[i:i + l])[::-1])

        odd_index_interleave = np.concatenate((odd_index_1,odd_adc_2),axis = 0)    # for concatenated pixels
        even_index_interleave = np.concatenate((even_index_1,even_adc_2),axis = 0)    # for concatenated pixels

        #odd_index_interleave = np.ravel(np.column_stack((odd_index_1,odd_adc_2)))    # for interleaved pixels
        #even_index_interleave = np.ravel(np.column_stack((even_index_1,even_adc_2)))    # for interleaved pixels

        #splitting the even part of the ADC since on the sensor they belong to the outer 256 x 2
        even_index_interleave_left = even_index_interleave[0:256]
        even_index_interleave_right = even_index_interleave[256:512]





        #splitting the even part of the ADC since on the sensor they belong to the outer 256 x 2
        odd_left_1 = odd_index_1[0:128]
        odd_right_1 = odd_index_1[128:256]

        odd_left_2 = odd_adc_2[0:128]
        odd_right_2 = odd_adc_2[128:256]


        #temp = [odd_right_1[i^1] for i in range(len(odd_right_1))]
        #odd_right_1 = temp

        #temp1 = [even_index_1[i^1] for i in range(len(even_index_1))]
        #even_index_1 = temp1


        #final_1024_index = np.concatenate((even_index_interleave_left,odd_index_interleave,even_index_interleave_right),axis = 0)
        #######################
        final_1024_index_concatenated = np.concatenate((odd_left_1,even_index_1,odd_right_1,odd_left_2,even_adc_2,odd_right_2),axis = 0)  # for concatenated pixels
        left =  np.concatenate((odd_left_1,even_index_1,odd_right_1),axis = 0)

        right = np.concatenate((odd_left_2,even_adc_2,odd_right_2),axis = 0)

        final_1024_index_interleaved = np.ravel(np.column_stack((left,right)))  #for interleaved pixels

        ##############-------------Reshape the data as pixel x orbits-------------------###################
        if (orbits):
            data = data[0:orbits*512*2]

        data = np.reshape(data,(-1,pixels))

        data = data[:,final_1024_index_interleaved]
        data = data >> 2
        data = ss.savgol_filter(data, 51, 3) # window size 51, polynomial order 3
        orb, pix = np.shape(data)

        return (data, orb)