DataManager.py 12.9 KB
Newer Older
Nick Sauerwein's avatar
Nick Sauerwein committed
1 2 3 4 5
import numpy as np
import datetime
import glob


6
import Globals as g
Nick Sauerwein's avatar
Nick Sauerwein committed
7 8 9

import matplotlib.pyplot as plt

Nick Sauerwein's avatar
Nick Sauerwein committed
10 11 12 13
def get_item_in_dict(dic, address):
    
    for step in address:
        dic = dic[step]
Nick Sauerwein's avatar
Nick Sauerwein committed
14
        
Nick Sauerwein's avatar
Nick Sauerwein committed
15
    return dic
Nick Sauerwein's avatar
Nick Sauerwein committed
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
def print_dict(dictionary, ident = '', braces=1, plot = True):
    """ Recursively prints nested dictionaries."""

    for key, value in dictionary.items():
        if isinstance(value, dict):
            print ('%s%s%s%s' %(ident,braces*'[',key,braces*']')) 
            print_dict(value, ident+'  ', braces+1)
        else:
            if type(value).__name__ == 'ndarray' and plot == True:
                print (ident+key+' (see plot below)' )
                if value.ndim == 1:
                    plt.figure()
                    plt.plot(value)
                    plt.title(key+'  level = '+str(braces))
                
                if value.ndim == 2:
                    from matplotlib import cm
                    plt.figure()
                    plt.imshow(value, cmap='gray')
                    plt.title(key+'  level = '+str(braces))
                    plt.colorbar()
            else:
                print (ident+'%s = %s' %(key, value))

40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170

                
def get_subfolders(base_folder):
    from glob import glob
    subfolders = []
    while True:
        folders = glob(base_folder+"*/")
        if folders == []:
            break
        subfolders += folders
        base_folder += '*/'
        
    return subfolders
        
                
def find_configs(base_folder, output = True):
    subfolders = get_subfolders(base_folder)
    
    configs = []
    
    for folder in subfolders:
        new_configs = IO(folder).search(output = False)
        for config in new_configs:
            config['folder'] = folder
        configs += new_configs
        
    return configs

def get_differences(dics, headids = []):
    
    if len(dics) < 2:
        return []
    
    
    
    
    ids = []
    
    ref_dic = dics[0]
    
    for dic in dics[1:]:
        for name, item in dic.items():
            if headids + [name] in ids:
                continue
            
            if not name in ref_dic:
                ids += [headids + [name]]
                continue
                
            if ref_dic[name] == item:
                continue
                
            if isinstance(item, dict):
                subdiff = get_differences(extract(dics,name),headids = headids + [name])
                for diff in subdiff:
                    if diff in ids:
                        continue
                    ids += [diff]
                    
                continue

            ids += [headids + [name]]      
                
            
    ref_dic = dics[-1]
    
    for dic in dics[:-1]:
        for name, item in dic.items():
            if headids + [name] in ids:
                continue
            
            if not name in ref_dic:
                ids += [headids + [name]]
                continue
                
            if ref_dic[name] == item:
                continue
                
            if isinstance(item, dict):
                subdiff = get_differences(extract(dics,name),headids = headids + [name])
                for diff in subdiff:
                    if diff in ids:
                        continue
                    ids += [diff]
                    
                continue

            ids += [headids + [name]]        
            
            
    return ids
        
        
        
        
def extract(measurements, list_of_ids):
    
    try:
        measurements = list(measurements)
    except:
        pass
    
    if type(list_of_ids) == str:
        list_of_ids = [list_of_ids]
        
    if type(measurements) is not list:
        measurements = [measurements]
        
    
    
    ms = measurements
    for i in list_of_ids:
        ms_new = []
        one_contains = False
        for m in ms:
            
            if isinstance(m,dict) and i in m:
                ms_new += [m[i]]
                one_contains = True
            else:
                ms_new += [None]
        if not one_contains:
            print (ms)
            ms_wo_none = filter(None.__ne__, ms)
            import itertools
            
            print ("id ",i," doesn't exsist. Choose from: ",set(itertools.chain(*[m.keys() for m in ms_wo_none])))
            assert (False)
        ms = ms_new
    return ms   
                
Nick Sauerwein's avatar
Nick Sauerwein committed
171
def print_data(data, plot = True):
172 173
        print_dict(data, plot = plot)
        
174
def plot_measure(data, devices, plot_config = {}, sleep = 0):
175 176 177 178 179 180
    if plot_config == {}:
        
        measure, config = (data['measure'], data['config'])
        for device in devices:
            device_name = type(device).__name__
            if hasattr(device, 'plot_measure'):
181
                device.plot_measure(measure[device_name], config = config[device_name])
182 183 184 185 186 187 188
        return
                    
    else:
        try:
            g.plot_hist
        except:
            g.plot_hist = {}
Nick Sauerwein's avatar
Nick Sauerwein committed
189 190 191 192 193
            
        try:
            g.plot_servers
        except:
            g.plot_servers = {}
194 195 196 197 198
        
          
        measure, config = (data['measure'], data['config'])
        for device in devices:
            device_name = type(device).__name__
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
199
            try:
200
                g.plot_hist[device_name]
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
201
            except:
202 203 204 205 206
                g.plot_hist[device_name] = {}

            for item in plot_config[device_name].items():
                if item[0] == 'plot_measure' :
                    if item[1]:
207
                        device.plot_measure(measure[device_name], config = config[device_name])
208
                else:
209 210
                    if not item[1]:
                        continue
211 212 213 214
                    try:
                        g.plot_hist[device_name][item[0]] =g.plot_hist[device_name][item[0]] + [measure[device_name][item[0]]]
                    except:
                        g.plot_hist[device_name][item[0]] = [measure[device_name][item[0]]]
Nick Sauerwein's avatar
Nick Sauerwein committed
215 216 217 218 219 220 221 222 223
                    
                    try:
                        ps1d = g.plot_servers['history: '+device_name+' '+item[0]]
                    except:
                        from PlottingServers import Plot1DServer 
                        ps1d = Plot1DServer({'xlabel' : 'last measurements','ylabel' : 'value', 'title': 'history: '+device_name+' '+item[0]})
                        g.plot_servers['history: '+device_name+' '+item[0]] = ps1d
                        

224
                    xx = range(-len(g.plot_hist[device_name][item[0]])+1,1)
Nick Sauerwein's avatar
Nick Sauerwein committed
225 226
                    ps1d.update(xx,g.plot_hist[device_name][item[0]])

227 228
                    
    #important: sort by time and plot sepecific elements
229
    
Nick Sauerwein's avatar
Nick Sauerwein committed
230 231 232 233 234 235 236 237 238 239 240 241 242

def dict_contains(d, item):
    for it in d.items():
        if isinstance(it[1], dict):
            if dict_contains(it[1], item):
                return True
        if it == item:
            return True
    return False

def set_config(devices, config):
    for device in devices:
        device_name = type(device).__name__
Hermann Benedikt's avatar
Hermann Benedikt committed
243 244 245 246
        try:
            device.set_config(config[device_name])
        except:
            print ('Device '+device_name+' is not in this config')
Nick Sauerwein's avatar
Nick Sauerwein committed
247 248 249 250 251 252
        
def get_config(devices):        
    time = str(datetime.datetime.now()).replace(':','-').replace(' ','_')
    config = {'time': time}
        
    for device in devices:
253 254
        import time
        t = time.time()
Nick Sauerwein's avatar
Nick Sauerwein committed
255 256
        device_name = type(device).__name__ 
        config[device_name] = device.get_config()
257
        print (device_name,' config time: ',time.time() - t)
Nick Sauerwein's avatar
Nick Sauerwein committed
258 259
    return config

root's avatar
root committed
260 261 262 263 264 265 266 267 268 269
def meas(dev):
    dn = type(dev).__name__
    if hasattr(dev, 'measure'):
        import time
        tt = time.time()
        d = dev.measure()
        print (dn,' measure time: ',time.time() - tt)
    return dn, d

def measure(devices, devices2measure = 'all', parallel = True):
Nick Sauerwein's avatar
Nick Sauerwein committed
270
    time = str(datetime.datetime.now()).replace(':','-').replace(' ','_')
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
271 272
    
    data = {'time': time}    
Nick Sauerwein's avatar
Nick Sauerwein committed
273

Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
274 275
    data['config'] = get_config(devices)
    data['measure'] = {}
276 277 278
    
    if devices2measure == 'all':
        devices2measure = devices
root's avatar
root committed
279 280 281 282 283 284 285 286 287 288
    if parallel:
        from joblib import Parallel, delayed
        res = Parallel(n_jobs= 4, backend="threading")(delayed(meas)(device) for device in devices2measure)
        
        for i in range(len(res)):
            data['measure'][res[i][0]]= res[i][1]
        
        return data
        
        
289
    for device in devices2measure:
Nick Sauerwein's avatar
Nick Sauerwein committed
290 291
        device_name = type(device).__name__
        if hasattr(device, 'measure'):
292 293
            import time
            t = time.time()
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
294
            data['measure'][device_name] = device.measure()
295
            print (device_name,' measure time: ',time.time() - t)
Nick Sauerwein's avatar
Nick Sauerwein committed
296
    
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
297
    return data     
Nick Sauerwein's avatar
Nick Sauerwein committed
298 299 300 301 302
    
class IO:
    
    def __init__(self, path):
        self.path = path
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
303 304 305 306 307 308 309 310 311
        try:
            import elog as el
            self.logbook = el.open('https://elog-gfa.psi.ch',
                                    'AMAS', user='sauerwein_n',
                                    password='Dark374Fields!',
                                    use_ssl=True, subdir='',
                                    encrypt_pwd=True)
        except:
            print ('This computer does not support elog')
312
    def save(self, data, name = None, elog = False, kind = 'config', comment = '', folder = ''):
Nick Sauerwein's avatar
Nick Sauerwein committed
313 314 315 316 317 318
        
        data['comment'] = comment

        if name == None:
            name = kind+data['time']

319
        np.save(self.path+folder+name+'.npy',data)
Nick Sauerwein's avatar
Nick Sauerwein committed
320
        print (kind+' saved as '+name+'.npy')
321 322 323 324 325 326 327 328 329 330 331 332
        
        if elog:
            
            
            class elogobj:
                def __init__(self,dic):
                    self.name = 'config.txt'
                    self.dic = dic
                def read(self):
                    return str(self.dic)
            
            obj = elogobj(data)
333
            import time
334 335 336 337
            self.logbook.post('Configuration with Comment: '+comment,
                         attributes={'Titel':'Configuration of LWFA',
                                     'Autor':'Sauerwein Nick Jacob (SQ84)',
                                     'Projekt': 'LPWA',
338 339
                                     'Dates': int(time.time())}, 
                                     attachments=[obj])
Nick Sauerwein's avatar
Nick Sauerwein committed
340 341
        return data
    
342 343 344
    def load(self, name_id = -1, kind = 'config', gui = False, elog = False):
        if elog:
            import requests
Sauerwein Nick Jacob's avatar
Sauerwein Nick Jacob committed
345 346 347
            response = requests.get(self.logbook.read(name_id)[-1][0],
                                    verify=False, 
                                    auth=('sauerwein_n', 'Dark374Fields!'))
348
            return eval(response.content.decode())
349
        if gui:
350
            from tkinter import Tk
351
            from tkinter.filedialog import askopenfilename
352 353 354 355 356
            root = Tk()
            root.withdraw()
            root.update()
            filename = askopenfilename(initialdir=self.path)
            root.distroy()
357
            return np.load(filename).item()
Nick Sauerwein's avatar
Nick Sauerwein committed
358 359 360 361
        if type(name_id) == int:
            return np.load([filen for filen in glob.iglob(self.path+kind+'*.npy')][name_id]).item()
        return np.load(self.path+str(name_id)+'.npy').item()
    
362
    def search(self, key_dict = {}, kind = 'config', output = True):
Nick Sauerwein's avatar
Nick Sauerwein committed
363
        datas = []
364 365
        if output:
            print ('n \t ',[item[0] for item in key_dict.items()])
Nick Sauerwein's avatar
Nick Sauerwein committed
366 367 368 369
        n = 0
        for filename in glob.iglob(self.path+kind+'*.npy'):
            data = np.load(filename).item()
            test = [dict_contains(data,item) for item in key_dict.items()]
370 371
            if output:
                print (n,' \t ',test)
Nick Sauerwein's avatar
Nick Sauerwein committed
372 373 374 375
            if np.all(test) or len(key_dict) == 0:
                datas += [data]
            n += 1
        return datas
Nick Sauerwein's avatar
Nick Sauerwein committed
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
    
    def select(self, keys = [{}], kind = 'measure'): 
        #example of key: {'address': ['where', 'to', 'find', 'data'], 'value': [val, 'b', error]}
        # 'b': bigger, 's': smaller, 'e': equals, in case of 'e' the error variable should be defined.
        results = []
        for filename in glob.iglob(self.path+kind+'*.npy'):
            data = np.load(filename).item()
            take = True
            for key in keys:
                try:
                    if key['value'][1] == 'b':
                        if key['value'][0] >= get_item_in_dict(data, key['address']):
                            take = False
                        print ('test')
                    if key['value'][1] == 's':
                        if key['value'][0] <= get_item_in_dict(data, key['address']):
                            take = False
                        
                    if key['value'][1] == 'e':
                        if key['value'][0] - key['value'][2] <= get_item_in_dict(data, key['address']) or key['value'][0] + key['value'][2] >= get_item_in_dict(data, key['address']):
                            take = False
                except:
                    take = False
                        
            if take:
                results += [data]
        return results
    
                        
            
            
        
        
Nick Sauerwein's avatar
Nick Sauerwein committed
409 410 411 412 413 414 415 416 417