KaptureControlGui/KCG/base/backend/CalibrationUpdater.py

import time
import os
import math
import numpy as np
from numpy.polynomial.polynomial import polyval
import h5py
import traceback
import csv
import warnings

from scipy.optimize import curve_fit
from scipy.stats import chisquare
from scipy.stats import chi2_contingency
from scipy.special import erfc, erf

import datetime
import pyqtgraph as pg
from PyQt4 import QtGui, QtCore, QtSvg

import matplotlib
matplotlib.use("Qt4Agg")
#matplotlib.use("Agg")
import matplotlib.pyplot as plt

try:
    #for KCG
    from ... import config
    from ...config import colours, coloursTrans
    from .. import kcgwidget as kcgw
    from .CalibrationHandle import theCalibration
    from .constants import KARA
    from .TimeScan import TimeScan
except:
    #for Analysis GUI
    import config
    from config import colours, coloursTrans
    import kcgwidget as kcgw
    from CalibrationHandle import theCalibration
    from constants import KARA
    from TimeScan import TimeScan


LINEAR = 0
GAUSS  = 1
GAUSSB = 2
EGAUSSC= 3
EGAUSS = 4
EGAUSSB= 5
SKEW   = 6
SINUS  = 7

import shutil

def sinus(x,a,b,c,d):
    return a*np.sin(b*x*(2*np.pi)+d)+c 

class CalibrationUpdater(kcgw.KCGWidgets):
    """
    The Timescan result plot subwindow.
    """
    def __init__(self, timeScan, parent=None, bunch=0, workingChannels=[0,1,2,3,4,5,6,7]):
        super(CalibrationUpdater, self).__init__()
        
        # ------[ Variable declaration ]------------
        self.parent=parent
        self.adc_number = 8
        self.timeScan = timeScan  # Data to plot
        self.calibId  = self.timeScan.calibId
        self.originalnotplotted = True
        self.running = False
        theCalibration.setHandle(self.calibId)
        self.grpX = theCalibration.grpX
        self.grpY = theCalibration.grpY

        self.workingChannels = workingChannels
        
        # -------[ Create structure ]----------
        self.outerLayout = QtGui.QVBoxLayout()  # Outermost layout of this window
        self.setLayout(self.outerLayout)

        self.settingsLayout = QtGui.QGridLayout()
        lineindex = 0
        
        fileName = self.timeScan.fileName
        if fileName is None:
            fileName = 'Last Measurement'
        self.text_label = QtGui.QLabel('selected Timescan: ' + fileName)
        self.settingsLayout.addWidget(self.text_label, lineindex, 0,1,2)
        lineindex +=1
        self.text_label = QtGui.QLabel('selected CalibrationFile: ' + self.calibId)
        self.settingsLayout.addWidget(self.text_label, lineindex, 0,1,2)
        lineindex +=1

        self.text_label = QtGui.QLabel('select type: ')
        self.settingsLayout.addWidget(self.text_label, lineindex, 0)
        self.comboBox = QtGui.QComboBox(self)
        self.comboBox.addItem("Peak Positive") #0
        self.comboBox.addItem("Peak Negative") #1
        self.comboBox.addItem("500MHz") #2
        self.settingsLayout.addWidget(self.comboBox, lineindex, 1)
        #lineindex +=1

        self.text_label = QtGui.QLabel('select Fit: ')
        self.settingsLayout.addWidget(self.text_label, lineindex, 2)
        self.fitSelect = QtGui.QComboBox(self)
        self.fitSelect.addItem("Exponential mod Gauss") #3
        self.fitSelect.addItem("Exponential mod Gauss V2") #4
        self.fitSelect.addItem("skew Gauss") #5
        self.settingsLayout.addWidget(self.fitSelect, lineindex, 3)
        lineindex +=1

        
        self.useFirst = QtGui.QSpinBox()
        self.useFirst.setRange(0, 1000)
        self.useFirst.setSingleStep(1)
        self.useFirst.setValue(0)
        self.settingsLayout.addWidget(QtGui.QLabel('use first data as Baseline'), lineindex, 0)
        self.settingsLayout.addWidget(self.useFirst, lineindex, 1)
        lineindex += 1

        self.multiBunchSelector = self.createSpinbox(0,184, start_value=bunch, interval=1, connect=self.on_updateBunch)
        self.multiBunchSelectorLabel = self.createLabel('Bunch Selection:')
        self.settingsLayout.addWidget(self.multiBunchSelectorLabel, lineindex, 0)
        self.settingsLayout.addWidget(self.multiBunchSelector, lineindex, 1)
        if self.timeScan.multibunch == False:
            self.multiBunchSelector.hide()
            self.multiBunchSelectorLabel.hide()
        lineindex += 1

        self.adcCheckBox = []
        self.adc_checkbox_layout = QtGui.QHBoxLayout()
        self.group = QtGui.QButtonGroup()
        self.group.setExclusive(False)
        #print(self.workingChannels)
        for i in range(self.adc_number):
            self.adcCheckBox.append(self.createCheckbox(text="ADC "+str(i+1), connect=self.changeWorking))
            self.adc_checkbox_layout.addWidget(self.adcCheckBox[i])
            self.group.addButton(self.adcCheckBox[i], i+1)
            if i in self.workingChannels:
                self.adcCheckBox[i].setChecked(True)
            else:
                self.adcCheckBox[i].setChecked(False)
        
        self.groupWidget = QtGui.QWidget()
        self.groupWidget.setLayout(self.adc_checkbox_layout)
        
        self.settingsLayout.addWidget(self.groupWidget, lineindex,0,1,4)
        lineindex +=1

        self.okay_btn = QtGui.QPushButton("Run")
        self.okay_btn.setStyleSheet("padding: 15px;")
        self.okay_btn.clicked.connect(self.on_okay)
        self.settingsLayout.addWidget(self.okay_btn, lineindex, 0)

        self.okay_btn = QtGui.QPushButton("Save")
        self.okay_btn.setStyleSheet("padding: 15px;")
        self.okay_btn.clicked.connect(self.on_save)
        self.settingsLayout.addWidget(self.okay_btn, lineindex, 1)
        
        self.cancel_btn = QtGui.QPushButton("Cancel")
        self.cancel_btn.setStyleSheet("padding: 15px;")
        self.cancel_btn.clicked.connect(self.on_cancel)
        self.settingsLayout.addWidget(self.cancel_btn, lineindex, 2)
        

        #######################

        self.plotLayout = QtGui.QHBoxLayout()
        self.plotWidget1 = QtGui.QWidget()
        self.plotlayout1= QtGui.QGridLayout()  # Main Layout to hold the most elements in this window      
        self.all_plot_widget = pg.PlotWidget(title="original")
        self.plotlayout1.addWidget(self.all_plot_widget, 0, 0)
        self.plotWidget1.setLayout(self.plotlayout1)

        self.plotWidget2 = QtGui.QWidget()
        self.plotlayout2 = QtGui.QGridLayout()  # Main Layout to hold the most elements in this window      
        self.calibrated_plot = pg.PlotWidget(title="calibrated")
        self.plotlayout2.addWidget(self.calibrated_plot, 0, 0)
        self.plotWidget2.setLayout(self.plotlayout2)

        self.plotLayout.addWidget(self.plotWidget1)
        self.plotLayout.addWidget(self.plotWidget2)

        # -- Create Parameter Adjust
        self.blocking = True
        
        self.parameterLayout = QtGui.QGridLayout()
        lineindex = 0
        self.adcBaselineBox = []
        self.parameterLayout.addWidget(self.createLabel('baseline'), lineindex,0)
        for i in range(self.adc_number):
            self.adcBaselineBox.append(self.createInput(text="", connect=self.update))
            self.parameterLayout.addWidget(self.adcBaselineBox[i], lineindex, i+1)
        lineindex += 1
            #self.adc_offset_layout.addWidget(self.createLabel('ADC '+ str(i+1)))
            #self.adc_offset_layout.addWidget(self.adcOffsetBox[i])

        self.adcScalingBox = []
        self.parameterLayout.addWidget(self.createLabel('scaling'), lineindex,0)
        for i in range(self.adc_number):
            self.adcScalingBox.append(self.createInput(text="", connect=self.update))
            self.parameterLayout.addWidget(self.adcScalingBox[i], lineindex, i+1)
        lineindex += 1  

        self.adcOffsetBox = []
        self.parameterLayout.addWidget(self.createLabel('offset (ps)'), lineindex,0)
        for i in range(self.adc_number):
            self.adcOffsetBox.append(self.createInput(text="", connect=self.update))
            self.parameterLayout.addWidget(self.adcOffsetBox[i], lineindex, i+1)
        lineindex += 1

        self.outerLayout.addLayout(self.settingsLayout)
        self.outerLayout.addLayout(self.plotLayout)
        self.outerLayout.addLayout(self.parameterLayout)
        self.blocking = False
        self.setWindowTitle("CalibrationUpdater")
        self.plot()
        self.displayParameter()


    def on_okay(self, param):
        if not self.running:
            self.old_id = self.calibId
            self.calibId += '.new'
            self.timeScan.calibId = self.calibId
            shutil.copy(self.old_id, self.calibId)
            theCalibration.openFile(self.calibId, write=True)
            self.grpX = theCalibration.grpX
            self.grpY = theCalibration.grpY
            self.running = True

        self.runCalibration()

    def on_save(self, param):
        if self.running:
            theCalibration.closeHandle(self.calibId)
            theCalibration.closeHandle(self.old_id)
            
            ctr=0
            filelist = np.sort(os.listdir(os.path.dirname(self.old_id)))
            for file in filelist:
                if '.old' in file:
                    if int(file[-2:]) >= ctr:
                        ctr = int(file[-2:])+1

            #print('rename')
            os.rename(self.old_id, self.old_id+'.old{:02d}'.format(ctr))
            os.rename(self.calibId, self.old_id)

            self.calibId = self.old_id
            theCalibration.openFile(self.calibId, force=True)
            self.timeScan.calibId = self.calibId
            self.running = False
        pass

    def on_cancel(self, param):
        
        self.close()
        pass

    def on_updateBunch(self, param):
        self.originalnotplotted = True
        self.plot()
        
    def changeWorking(self, param):
        self.workingChannels = []
        for i in range(self.adc_number):
            if self.adcCheckBox[i].isChecked():
                self.workingChannels.append(i)
        #print(self.workingChannels)
        self.originalnotplotted = True
        self.plot()

    def displayParameter(self):
        self.blocking = True
        for i in self.workingChannels:
            try:
                popt = self.grpY[str(i)][0]
                self.adcBaselineBox[i].setText('{:.3f}'.format(popt[0]))
                self.adcScalingBox[i].setText('{:.3f}'.format(popt[1]))
                self.adcOffsetBox[i].setText('{:.3f}'.format(self.grpX[str(i)]['offset'][()]*1e12))
            except:
                pass
        
        self.blocking = False


    def update(self):
        if not self.running:
            return
        if self.blocking:
            return
        for i in self.workingChannels:      
            #print(self.adcOffsetBox[i].text())
            #print(self.grpY[str(i)][0])
            self.grpY[str(i)][...] = [[float(str(self.adcBaselineBox[i].text())),  float(str(self.adcScalingBox[i].text()))]]
            self.grpX[str(i)]['offset'][...] = float(str(self.adcOffsetBox[i].text()))*1e-12
            #print(i, self.grpY[str(i)][0], self.grpX[str(i)]['offset'][()])
        pass
        self.plot()


    def plot(self):
        """
        Plot Data
        :return: -
        """  

        self.processbarWidget = WaitingWidget(self)
        if self.originalnotplotted:
            self.originalnotplotted = False
            self.all_plot_widget.plotItem.clear()
            for i in self.workingChannels:
                try:
                    scanx, scany, scane = self.timeScan.getScanOverTime(i, False, False, bunch=self.multiBunchSelector.value())
                    self.all_plot_widget.plotItem.plot(scanx, scany, pen=None, symbolPen=pg.mkPen(colours[i]), symbolSize=4, pxMode=True, symbolBrush=pg.mkBrush(colours[i]), downsample=200)
                except:
                    pass
                

        
        self.calibrated_plot.plotItem.clear()
        for i in self.workingChannels:
            try:
                scanx, scany, scane = self.timeScan.getScanOverTime(i, True, True, force=i==0, bunch=self.multiBunchSelector.value())
                
                sorter = np.argsort(scanx)
                scanx = scanx[sorter]
                scany = scany[sorter]
                N=3
                scanyAvg = np.convolve(scany, np.ones((N,))/N, mode='valid') 

                self.calibrated_plot.plotItem.plot(scanx, scany, pen=None, symbolPen=pg.mkPen(colours[i]), symbolSize=4, pxMode=True, symbolBrush=pg.mkBrush(colours[i]), downsample=200)
                self.calibrated_plot.plotItem.plot(scanx[1:-1], scanyAvg, pen=pg.mkPen(colours[i]), downsample=200)
            except:
                pass
                
        self.processbarWidget.close()
        self.processbarWidget = None
        self.show()
        self.setFocus()

    ##################################################################################
    def closeEvent(self, event):
        """
        Event handler for closing this window
        """
        #reopen file so that it is in read only mode
        if self.running:
            theCalibration.closeHandle(self.calibId)
            self.timeScan.calibId = self.old_id
            self.calibId = self.old_id

        theCalibration.openFile(self.calibId, force=True)
        if self.processbarWidget is not None:
            self.processbarWidget.close()
        if self.parent is not None:
            self.parent.updater = None
        




#  .d8888b.                                              888                  .d8888b.           888 d8b 888                       888    d8b
# d88P  Y88b                                             888                 d88P  Y88b          888 Y8P 888                       888    Y8P
# 888    888                                             888                 888    888          888     888                       888
# 888         .d88b.  88888b.   .d88b.  888d888  8888b.  888888  .d88b.      888         8888b.  888 888 88888b.  888d888  8888b.  888888 888  .d88b.  88888b.
# 888  88888 d8P  Y8b 888 "88b d8P  Y8b 888P"       "88b 888    d8P  Y8b     888            "88b 888 888 888 "88b 888P"       "88b 888    888 d88""88b 888 "88b
# 888    888 88888888 888  888 88888888 888     .d888888 888    88888888     888    888 .d888888 888 888 888  888 888     .d888888 888    888 888  888 888  888
# Y88b  d88P Y8b.     888  888 Y8b.     888     888  888 Y88b.  Y8b.         Y88b  d88P 888  888 888 888 888 d88P 888     888  888 Y88b.  888 Y88..88P 888  888
#  "Y8888P88  "Y8888  888  888  "Y8888  888     "Y888888  "Y888  "Y8888       "Y8888P"  "Y888888 888 888 88888P"  888     "Y888888  "Y888 888  "Y88P"  888  888
#
#
#
    def runCalibration(self):
        typ = self.comboBox.currentIndex()
        fittype =  self.fitSelect.currentIndex()+EGAUSS
        useFirst = self.useFirst.value()

        #print(typ, fittype, useFirst)
        
        if typ in [0,1]: #Peak
            self.updateY(typ, fittype, useFirst)
        elif typ == 2: #500MHz
            self.updateX()
            #self.generateMap(fh)
            pass
        pass


# Y88b   d88P      .d8888b.           888 d8b 888                       888    d8b
#  Y88b d88P      d88P  Y88b          888 Y8P 888                       888    Y8P
#   Y88o88P       888    888          888     888                       888
#    Y888P        888         8888b.  888 888 88888b.  888d888  8888b.  888888 888  .d88b.  88888b.
#     888         888            "88b 888 888 888 "88b 888P"       "88b 888    888 d88""88b 888 "88b
#     888         888    888 .d888888 888 888 888  888 888     .d888888 888    888 888  888 888  888
#     888         Y88b  d88P 888  888 888 888 888 d88P 888     888  888 Y88b.  888 Y88..88P 888  888
#     888          "Y8888P"  "Y888888 888 888 88888P"  888     "Y888888  "Y888 888  "Y88P"  888  888
#
#
#

    def updateY(self, isNegativ=0, fittype=EGAUSSB, useFirst=0):
        
        try:
            self.generateYCalibration(theCalibration.fileHandle, isNegativ, fittype, useFirst)
        except:
            print('error Calibrating Y')
            traceback.print_exc()
                      
        #self.generateMap(fh)
        self.displayParameter()
        self.plot()

    def generateYCalibration(self, fh, isNegativ=0, fittype=EGAUSSB, useFirst=0):
        print('start Y Calibration')
        grpY = fh['y']
        grpX = fh['x']

        fitFun = lambda x: x
        phase0 = 0.0
        A0 =0.0

        for i in range(self.timeScan.adcNumber):
            try:
                grpX[str(i)]['offset'][...] = 0
            except:
                try:
                    grpX[str(i)]['offset'] = 0
                except:
                    pass


        for i in range(self.timeScan.adcNumber):  
            print(i,'-----------------------------')          
            baseline = 0.0

            if i not in self.workingChannels:
                grpY[str(i)][...] = np.array([[0.0,1.0]])
                #grpY.create_dataset(str(i), data=np.array([[0.0,1.0]]))
                #grpX[str(i)]['offset'] = 0
                continue

            try:
                
                #scanx = self.prepareX(data, grpX['d330'], grpX['d25'], grpX['d3'], grpX[str(i)]['330'], grpX[str(i)]['25'], grpX[str(i)]['3'], zero25B=grpX['d25b_zero'], cal25B=grpX[str(i)]['25b'])
                scanx, scany, scane = self.timeScan.getScanOverTime(i, True, False, force=i==0, bunch=self.multiBunchSelector.value())
                if useFirst:
                    y = scany[np.argsort(scanx)]
                    baseline = np.mean(y[:useFirst])
                    print('baseline ', baseline)
                #print(scanx.shape, scany.shape)
                popt, perr, fitFun, pcov, label, x = self.doFit(scanx, scany, fittype, baseline=baseline, findMin=isNegativ)
                phase, A = self.findMax(scanx, popt, fitFun, findMin=isNegativ)
            except:
                #try:
                #    popt, perr, fitFun, pcov, label, x = self.doFit(scanx, scany[i], EGAUSSB, baseline=baseline)
                #    phase, A = self.findMax(scanx, popt, fitFun, findMin=isNegativ)
                #except:
                try:
                    grpY[str(i)][...] = np.array([[0.0,1.0]])
                except:
                    print(i)
                    traceback.print_exc()
                    grpY[str(i)] = np.array([[0.0,1.0]])
                traceback.print_exc()
                print(i,' not fit')
                continue



            if phase0 == 0:
                phase0 = phase
            grpX[str(i)]['offset'][...] = phase0 - phase
            #print(phase, A, popt[4])
            A -= popt[4]
            if A0 == 0:
                A0 = A
            if isNegativ:
                A *= -1
            #print(phase, phase0 - phase, A)
            try:
                grpY[str(i)][...] = data=np.array([[popt[4], A/A0]], dtype=np.float64)
                grpY['{}popt'.format(i)][...] = popt
                grpY['{}baseline'.format(i)][...] = baseline
            except:
                grpY[str(i)] = data=np.array([[popt[4], A/A0]], dtype=np.float64)
                grpY['{}popt'.format(i)] = popt
                grpY['{}baseline'.format(i)] = baseline


            
        print('generateYCalibration done')


     
    # Y88b   d88P      .d8888b.           888 d8b 888                       888    d8b
    #  Y88b d88P      d88P  Y88b          888 Y8P 888                       888    Y8P
    #   Y88o88P       888    888          888     888                       888
    #    Y888P        888         8888b.  888 888 88888b.  888d888  8888b.  888888 888  .d88b.  88888b.
    #    d888b        888            "88b 888 888 888 "88b 888P"       "88b 888    888 d88""88b 888 "88b
    #   d88888b       888    888 .d888888 888 888 888  888 888     .d888888 888    888 888  888 888  888
    #  d88P Y88b      Y88b  d88P 888  888 888 888 888 d88P 888     888  888 Y88b.  888 Y88..88P 888  888
    # d88P   Y88b      "Y8888P"  "Y888888 888 888 88888P"  888     "Y888888  "Y888 888  "Y88P"  888  888
    #
    #
    #
    def updateX(self):
        
        try:
            self.generateXCalibration(theCalibration.fileHandle)
            self.generateMap(theCalibration.fileHandle)
        except:
            
            print('error Calibrating X')
            traceback.print_exc()
            return    

        print('all Done')
        self.displayParameter()
        self.plot()
        #self.plotWidget.finetune(peakScanFile, self.fileHandle)

    def generateXCalibration(self, fh):
        grpX = fh['x']

        self.processbarWidget = ProcesbarWidget(self, True)
        valmax = 100
        val = 0

        verbose = False

        doPlots = False

        try:
            grpX['d330'][...] = 334
            grpX['d25'][...]  = 23
            grpX['d3'][...]   = 3
            grpX['d25b'][...] = 23
            grpX['d25b_zero'][...]   = 4
        except:
            grpX['d330'] = 334
            grpX['d25']  = 23
            grpX['d3']   = 3
            grpX['d25b'] = 23
            grpX['d25b_zero']   = 4
            
        stepswith25 = 13
        stepswith3  = 32
        zero25 = 0

        steps = [0,1,3,4,6,7]
        data  = self.timeScan.scanData
        adc   = self.timeScan.adcNumber
        
        ### X Calibration:

        print('i |       A      |       F      |       b      |       phi')
        phase0 = 0
        for i in range(adc):
            if i in [4,5]:
                continue

            self.processbarWidget.update(val/valmax*1000)
            val += 1
            try:
                adcgrp = grpX[str(i)]
            except:
                adcgrp = grpX.create_group(str(i))
            cal33 = None
            cal25 = None
            cal3  = None
            popt0 = None
            popt  = 0
            fitx0 = 0
            phase = 0
            for iteration in range(1):
                scanx, scany = self.prepareScan(data, grpX['d330'], grpX['d25'], grpX['d3'],cal33,cal25,cal3, zero25B=grpX['d25b_zero'])
                fitx = scanx

                popt, perr = self.fit500(fitx[::32], scany[i][::32])
                phase = popt[3]/(2*np.pi)/popt[1]#1e3#*1e9
                if popt0 is None:
                    popt0 = popt
                    fitx0 = fitx
                
                if i == 0:
                    phase0 = phase

                self.processbarWidget.update(val/valmax*1000)
                val += 1
                

                print(i, popt, phase0 -phase, iteration)
                
        
                ######################################
                # Calib 330ps
                print('Calib 330ps')
                scanx, scany = self.prepareScan(data, grpX['d330'], grpX['d25'], grpX['d3'], defaultB=False, zero25B=grpX['d25b_zero'])
                if verbose:
                    print(scanx.shape)
                    print(scany.shape)
                    print(len(data[0]))

                tmp = np.ones((14,stepswith25,stepswith3))
                for j in range(len(data[0])):
                    #data[0][j][2] ist 330ps; [][][3] ist 25ps; [][][4] ist 3ps; [][][6] ist 25ps FCM2
                    if int(data[0][j][3]) == zero25 and  int(data[0][j][4]) == 0 and int(data[0][j][6]) == int(grpX['d25b_zero'][()]):
                        tmp[int(data[0][j][2])][int(data[0][j][3])][int(data[0][j][4])] = self.findDeltaX(scanx[j], scany[i][j], 0.1, popt, 1)
                dat33 = tmp[:,zero25,0]
                cal33 = dat33[np.where(dat33[:] != 1)]*1e12
                
                if verbose: print('call33 {}'.format(cal33))

                self.processbarWidget.update(val/valmax*1000)
                val += 1
                
                # Plot
                if doPlots:
                    plt.figure()
                    if iteration > 0:
                        plt.plot(np.sort(fitx0), sinus(np.sort(fitx0),*popt0))
                    plt.plot(np.sort(fitx), sinus(np.sort(fitx),*popt))
                    scanx, scany = self.prepareScan(data, grpX['d330'], grpX['d25'], grpX['d3'], zero25B=grpX['d25b_zero'])
                    plt.plot(scanx[::33*stepswith25], scany[i][::33*stepswith25], '.', label='{} org'.format(iteration))
        
                    scanx, scany = self.prepareScan(data, grpX['d330'], grpX['d25'], grpX['d3'], cal33, zero25B=grpX['d25b_zero'])
                    plt.plot(scanx[::33*stepswith25], scany[i][::33*stepswith25], 'x', label='{} x'.format(iteration))
                    plt.legend()
                    #plt.show()
                

                ######################################
                # Calib 25ps
                print('Calib 25ps')
                data2 = self.selectScan(data, steps)
                scanx, scany = self.prepareScan(data2, grpX['d330'], grpX['d25'], grpX['d3'], cal33, defaultB=False, zero25B=grpX['d25b_zero'])

                tmp = np.ones((14,stepswith25,stepswith3))
                l = len(data2[0])
                for j in range(len(data2[0])):
                    if int(data2[0][j][4]) == 0 and int(data2[0][j][6]) == grpX['d25b_zero'][()]:
                        if j%2 == 0:
                            self.processbarWidget.update(val/valmax*1000, (j*1.0)/l*1000.0)
                        tmp[int(data2[0][j][2])][int(data2[0][j][3])][int(data2[0][j][4])] = self.findDeltaX(scanx[j], scany[i][j], 0.5, popt,1)
                dat25 = tmp[steps,:,0]
                dat25 = dat25[np.where(dat25[:,0] != 1)]*1e12
                
                cal25 = np.mean(dat25,0)
                if verbose: print('cal25 {}'.format(cal25))
                self.processbarWidget.update(val/valmax*1000)
                val += 1

                
                # Plot
                if doPlots:
                    plt.figure()
                    if iteration > 0:
                        plt.plot(np.sort(fitx0), sinus(np.sort(fitx0),*popt0))
                    plt.plot(np.sort(fitx), sinus(np.sort(fitx),*popt))
                    scanx, scany = self.prepareScan(data2, grpX['d330'], grpX['d25'], grpX['d3'], zero25B=grpX['d25b_zero'])
                    plt.plot(scanx[::33], scany[i][::33], '1', label='org {}'.format(iteration))
                    scanx, scany = self.prepareScan(data2, grpX['d330'], grpX['d25'], grpX['d3'], cal33, zero25B=grpX['d25b_zero'])
                    plt.plot(scanx[::33], scany[i][::33], '2', label='cal330 {}'.format(iteration))
                    scanx, scany = self.prepareScan(data2, grpX['d330'], grpX['d25'], grpX['d3'], cal33, cal25, zero25B=grpX['d25b_zero'])
                    plt.plot(scanx[::33], scany[i][::33], '3', label='cal 330+25 {}'.format(iteration))
                    plt.legend()
                    #plt.show()
                
                self.processbarWidget.update(val/valmax*1000)
                val += 1
                ######################################
                # Calib 3ps
                print('Calib 3ps')
                scanx, scany = self.prepareScan(data2, grpX['d330'], grpX['d25'], grpX['d3'], cal33, defaultB=False, zero25B=grpX['d25b_zero'])

                tmp = np.ones((14,stepswith25,stepswith3))
                l = len(data2[0])
                for j in range(len(data2[0])-1):
                    if int(data2[0][j][2]) in steps and int(data2[0][j][6]) == grpX['d25b_zero'][()]:
                        if j%8 == 0:
                            self.processbarWidget.update(val/valmax*1000, (j*1.0)/l*1000.0)
                        tmp[int(data2[0][j][2])][int(data2[0][j][3])][int(data2[0][j][4])] = self.findDeltaX(scanx[j], scany[i][j], 1, popt,1)
                dat3 = tmp[steps,:12,:]
                dat3 = dat3[np.where(dat3[:,0,1] != 1)]*1e12
                if len(dat3) == 0:
                    cal3 = np.zeros(stepswith3)
                else:
                    cal3 = np.mean(np.mean(dat3, 0),0)


                if verbose: print(cal3.shape)
                print('done')

                if doPlots:
                    print('plot')
                    plt.figure()
                    if iteration > 0:
                        plt.plot(np.sort(fitx0), sinus(np.sort(fitx0),*popt0))
                    plt.plot(np.sort(fitx), sinus(np.sort(fitx),*popt))
                    scanx, scany = self.prepareScan(data, grpX['d330'], grpX['d25'], grpX['d3'], zero25B=grpX['d25b_zero'])
                    plt.plot(scanx, scany[i], '1', label='org {}'.format(iteration))
                    scanx, scany = self.prepareScan(data, grpX['d330'], grpX['d25'], grpX['d3'], cal33, cal25, cal3, zero25B=grpX['d25b_zero'])
                    plt.plot(scanx, scany[i], '3', label='cal33+25+3 {}'.format(iteration))
                    plt.legend()
                    plt.plot()
            

                self.processbarWidget.update(val/valmax*1000)
                val += 1

                #Iteration End

            if doPlots:
                plt.show()
            
            try:
                adcgrp['fit_popt'][...] = popt
                adcgrp['fit_perr'][...] = perr

                adcgrp['330'][...] = cal33
                adcgrp['25'][...]  = cal25
                adcgrp['3'][...]   = cal3
            except:
                adcgrp['offset'] = phase0 - phase
                adcgrp['fit_popt'] = popt
                adcgrp['fit_perr'] = perr
                adcgrp.create_dataset('330', data=cal33)
                adcgrp.create_dataset('25', data=cal25)
                adcgrp.create_dataset('3', data=cal3)
            

            self.processbarWidget.update(val/valmax*1000)
            val += 1
            ######################################
            # Calib 25ps FMC2
            dat25b=np.zeros(8)
            if i < 4: #only to be done for ADC1-4 on FMC2
                print('Calib FMC2')
                scanx, scany = self.prepareScan(data2, grpX['d330'], grpX['d25'], grpX['d3'], cal33, cal25, cal3, defaultB=False, zero25B=grpX['d25b_zero'])
                tmp = np.zeros(12)
                ctr = np.zeros(12)
                for j in range(len(data2[0])):
                    if int(data2[0][j][6]) != grpX['d25b_zero']:
                        if j%2 == 0:
                            self.processbarWidget.update(val/valmax*1000, (j*1.0)/l*1000.0)
                        tmp[int(data2[0][j][6])] += self.findDeltaX(scanx[j], scany[i][j], 1, popt,1)
                        ctr[int(data2[0][j][6])] += 1
                dat25b = tmp[np.where(tmp != 1)]*1e12
                ctr[np.where(ctr==0)] += 1
                
                dat25b = dat25b/ctr
                tmp[grpX['d25b_zero']] = 0
                
            try:
                adcgrp['25b'][...] = dat25b
            except:
                adcgrp.create_dataset('25b', data=dat25b)

            
        
        if self.processbarWidget is not None:
            self.processbarWidget.close()
            self.processbarWidget = None

    #############################
    ### Generate time Map
    def generateMap(self, fh):
        print('generateMap')
        grpX = fh['x']
        self.processbarWidget = ProcesbarWidget(self)
        val = 0
        valmax = 8*8*13 #*32*9

        delays = np.zeros((8, 8, 13, 32, 9))
        for adc in range(8):
            for c33 in range(0,7+1):
                for c25 in range(13):
                    for f in range(32):
                        for c25b in range(9):
                            x = c33*grpX['d330'][()] + c25*grpX['d25'][()] + f*grpX['d3'][()]
                            if adc < 4:
                                x -= (c25b - grpX['d25b_zero'][()]) * grpX['d25b'][()]
                            x = x*1e-12
                            if adc in [4,5]:
                                delays[adc][c33][c25][f][c25b] = x
                                continue
                            corr=  (grpX[str(adc)]['330'][c33] + 
                                    grpX[str(adc)]['25'][c25] + 
                                    grpX[str(adc)]['3'][f])
                            if adc < 4:
                                corr += grpX[str(adc)]['25b'][c25b][()]
                            corr = corr*1e-12

                            #corr += grpX[str(adc)]['offset'][()]
                            x += corr
                            delays[adc][c33][c25][f][c25b] = x
                    if self.processbarWidget is not None:
                        self.processbarWidget.update(val/valmax*1000)
                    val +=1

        try:
            grpX['map'][...] = delays
        except:
            processbarWidgetgrpX['map'] = delays
        if self.processbarWidget is not None:
            self.processbarWidget.close()
            self.processbarWidget = None
        print('done')
        

    """

    # 888    888          888                               8888888888                         888    d8b
    # 888    888          888                               888                                888    Y8P
    # 888    888          888                               888                                888
    # 8888888888  .d88b.  888 88888b.   .d88b.  888d888     8888888 888  888 88888b.   .d8888b 888888 888  .d88b.  88888b.  .d8888b
    # 888    888 d8P  Y8b 888 888 "88b d8P  Y8b 888P"       888     888  888 888 "88b d88P"    888    888 d88""88b 888 "88b 88K
    # 888    888 88888888 888 888  888 88888888 888         888     888  888 888  888 888      888    888 888  888 888  888 "Y8888b.
    # 888    888 Y8b.     888 888 d88P Y8b.     888         888     Y88b 888 888  888 Y88b.    Y88b.  888 Y88..88P 888  888      X88
    # 888    888  "Y8888  888 88888P"   "Y8888  888         888      "Y88888 888  888  "Y8888P  "Y888 888  "Y88P"  888  888  88888P'
    #                         888
    #                         888
    #                         888
    """
    def prepareX(self, data, a=330,b=25,c=3, cal33=None, cal25=None, cal3=None, steps=[0,3,6,9,12], zero25B=4, cal25B=None):
    
        if cal33 is None:
            scanx = np.array([val[2]*a                    + val[3]*b + val[4]*c + (zero25B - val[6])*b + (cal25B[int(val[6])] if cal25B is not None else 0) for val in data[0]])*1e-12
        elif cal25 is None:
            scanx = np.array([val[2]*a+cal33[int(val[2])] + val[3]*b + val[4]*c + (zero25B - val[6])*b + (cal25B[int(val[6])] if cal25B is not None else 0) for val in data[0]])*1e-12
        elif cal3 is None:
            scanx = np.array([val[2]*a+cal33[int(val[2])] + 
                              val[3]*b+cal25[int(val[3])] +            val[4]*c + (zero25B - val[6])*b + (cal25B[int(val[6])] if cal25B is not None else 0) for val in data[0]])*1e-12
        else:
            scanx = np.array([val[2]*a+cal33[int(val[2])] + 
                              val[3]*b+cal25[int(val[3])] + 
                              val[4]*c+cal3[int(val[4])]  +                       (zero25B - val[6])*b + (cal25B[int(val[6])] if cal25B is not None else 0) for val in data[0]])*1e-12

        return scanx

    def prepareScan(self, data, a=330,b=25,c=3, cal33=None, cal25=None, cal3=None, steps = [0,3,6,9,12], defaultB=True, zero25B=4, cal25B=None):
        scany = []
        tmp = []
        #print(data.shape)
        for i in range(len(data)):
            y = data[i].T[0]
            if defaultB:
                y = y[np.where(data[0,:,6] == zero25B)]
            scany.append(y)

        scany = np.array(scany)
        scanx = self.prepareX(data,a,b,c, cal33, cal25, cal3, steps, zero25B, cal25B)
        if defaultB:
            scanx = scanx[np.where(data[0,:,6] == zero25B)]
            
        return scanx, scany

    def selectScan(self, data, selector):
        #selector = [0,1,3,4,6,7]

        erg = [[],[],[],[], [],[],[],[]] 
        for i in range(len(data[0])):
            if data[0][i][2] in selector:
                for j in range(self.timeScan.adcNumber):
                    erg[j].append(data[j][i])

        return np.array(erg)
    
    def findMax(self, scanx, popt, fitFun, findMin=False):
        x = np.linspace(np.min(scanx), np.max(scanx), 100000)
        y = fitFun(x, *popt)
        if not findMin:
            return x[np.where(y==np.max(y))][0], np.max(y)
        else:
            return x[np.where(y==np.min(y))][0], np.min(y)

    def fit500(self, x, y):       
        p0 = (800, 500e6, 2048, 4.5)
        label = ['  A  ', '  f  ', '  b  ', '  phi']
        
        popt, pcov = curve_fit(sinus, x, y, p0=p0)
        #popt[0] = np.max(y)-popt[2]
        perr = np.sqrt(np.absolute(np.diag(pcov)))

        return popt, perr

    def findDeltaX(self, x, y, tol, popt, i):

        tmpx = np.linspace(x-50e-12, x+50e-12, 500000)
        tmpy = sinus(tmpx, *popt)
        
        sel = np.where((tmpy < y+tol) & (tmpy > y-tol))
        res = tmpx[sel]
        tmpy = tmpy[sel]
        tmp = res
        while len(tmp) > 4:
            res=tmp
            tol = tol*0.4
            sel = np.where((tmpy < y+tol) & (tmpy > y-tol))
            tmp = tmp[sel]
            tmpy = tmpy[sel]
            #if i%(24) == 0: print('tol ', tol, len(res))

        if len(res) > 1:
            res = [res[len(res)//2]]
        if len(res) == 0:
            print('bad',x, y)
            res= [x]
        
        return res[0]-x

    def doFit(self, x,y, type, p0=None, baseline=0, findMin=False):
        #np.seterr(all='warn')
        #warnings.filterwarnings('error')

        def linear(x,a,b):
            return a + b*x

        def gauss(x, sigma, mu, A, b):
            return A*np.exp(-0.5*((x-mu)/sigma)**2) + b

        def expgauss(x, s, m, a ,l, b):
            return a*l/2*np.exp(l/2*(2*m + l*s**2 - 2*x)) * erfc((m +l*s**2 -x)/(np.sqrt(2)*s)) + b

        def expgauss2(x, s, m, a, t, b):
            return a*s/t*np.sqrt(np.pi/2) * np.exp(0.5 * (s/t)**2 - (x-m)/t) * erfc(1/np.sqrt(2)*(s/t - (x-m)/s)) + b
        
        def sinus(x,a,b,c,d):
            return a*np.sin(b*x*(2*np.pi)+d)+c 

        def skewnorm(x, s, m, a, l, b):
            def p(x):
                return 1/np.sqrt(2*np.pi) * np.exp(- x**2/2)
            def g(x):
                return 0.5*(1+erf(x/np.sqrt(2)))

            return a*2/s*p((x-m)/s)*g(l*(x-m)/s) + b
        
        if findMin:
            startx = x[np.where(y==np.min(y))]
        else:
            startx = x[np.where(y==np.max(y))]

        y = y[np.argsort(x)]
        x = x[np.argsort(x)]

        if type == EGAUSS:
            fitFun = expgauss
            p0 = (23e-12, startx, 7e-8, 0.02e12, 2040)
            if findMin:
                p0 = (23e-12, startx, -7e-8, 0.02e12, 2040)
            label = ['  s  ', '  m  ', '  a  ', '  l  ', '  b  ']
        elif type == EGAUSSB:
            fitFun = expgauss2
            p0 = [19e-12, startx, 1200, 40e-12, 2040]
            if findMin:
                p0 = [19e-12, startx, -1200, 40e-12, 2040]
            label = ['  s  ', '  m  ', '  a  ', '  l  ', '  b  ']
        elif type == SKEW:
            fitFun = skewnorm
            p0 = [80e-12, startx, 6e-8, 3, 2040] # 220e-12
            if findMin:
                p0 = [80e-12, startx, -6e-8, 3, 2040] # 220e-12
            label = ['  s  ', '  m  ', '  a  ', '  l  ', '  b  ']
        else:
            print('Fit type unknown ', type)
            return 0,0, lambda x: x , 0

        
        if baseline != 0:
            print('baseline', baseline)
            fitFunOrg = fitFun
            fitFun = lambda x, s, m, a, l: fitFunOrg(x,s,m,a,l, baseline)
            p0 = p0[:-1]

        if p0 is not None:
            popt, pcov = curve_fit(fitFun, x, y, p0=p0)
        else:
            popt, pcov = curve_fit(fitFun, x, y)
        #popt = p0
        perr = np.sqrt(np.absolute(np.diag(pcov)))

        if baseline != 0:
            fitFun = fitFunOrg
            popt = np.append(popt, baseline)

        return popt, perr, fitFun, pcov, label, x






# 8888888b.                                              888                       888       888 d8b      888                   888
# 888   Y88b                                             888                       888   o   888 Y8P      888                   888
# 888    888                                             888                       888  d8b  888          888                   888
# 888   d88P 888d888  .d88b.   .d8888b  .d88b.  .d8888b  88888b.   8888b.  888d888 888 d888b 888 888  .d88888  .d88b.   .d88b.  888888
# 8888888P"  888P"   d88""88b d88P"    d8P  Y8b 88K      888 "88b     "88b 888P"   888d88888b888 888 d88" 888 d88P"88b d8P  Y8b 888
# 888        888     888  888 888      88888888 "Y8888b. 888  888 .d888888 888     88888P Y88888 888 888  888 888  888 88888888 888
# 888        888     Y88..88P Y88b.    Y8b.          X88 888 d88P 888  888 888     8888P   Y8888 888 Y88b 888 Y88b 888 Y8b.     Y88b.
# 888        888      "Y88P"   "Y8888P  "Y8888   88888P' 88888P"  "Y888888 888     888P     Y888 888  "Y88888  "Y88888  "Y8888   "Y888
#                                                                                                                  888
#                                                                                                             Y8b d88P
#                                                                                                              "Y88P"

class ProcesbarWidget(kcgw.KCGWidgets):
    """
    The Timescan result plot subwindow.
    """
    def __init__(self, parent, twobars=False):
        super(ProcesbarWidget, self).__init__()
        self.setWindowTitle("Running...")
        
        # ------[ Variable declaration ]------------
        self.parent = parent
        self.twobars = twobars
        

        # -------[ Create structure ]----------
        self.outerLayout = QtGui.QVBoxLayout()  # Outermost layout of this window
        self.setLayout(self.outerLayout)
        self.progressbar = QtGui.QProgressBar()
        #self.layout.addWidget(self.progressbar, lineindex,0,1,4)
        self.progressbar.setRange(0,1000)

        self.outerLayout.addWidget(self.createLabel('    ....Running....    '))
        self.outerLayout.addWidget(self.progressbar)
        if twobars:
            self.progressbar2 = QtGui.QProgressBar()
            #self.layout.addWidget(self.progressbar, lineindex,0,1,4)
            self.progressbar2.setRange(0,1000)
            self.outerLayout.addWidget(self.progressbar2)


        self.show()

    def update(self, value, value2=0):
        self.progressbar.setValue(value)
        if self.twobars:
            self.progressbar2.setValue(value2)
        QtGui.qApp.processEvents()

    ##################################################################################
    def closeEvent(self, event):
        """
        Event handler for closing this window
        """
        self.parent.processbarWidget = None
        pass


class WaitingWidget(kcgw.KCGWidgets):
    """
    The Timescan result plot subwindow.
    """
    def __init__(self, parent):
        super(WaitingWidget, self).__init__()
        self.setWindowTitle("please wait")
        
        # ------[ Variable declaration ]------------
        self.parent = parent
        

        # -------[ Create structure ]----------
        self.outerLayout = QtGui.QVBoxLayout()  # Outermost layout of this window
        self.setLayout(self.outerLayout)

        self.outerLayout.addWidget(self.createLabel('        ....Loading....        '))

        self.show()
        QtGui.qApp.processEvents()


    ##################################################################################
    def closeEvent(self, event):
        """
        Event handler for closing this window
        """
        self.parent.processbarWidget = None
        pass