# -*- coding: utf-8 -*- """ Created on Mon Aug 10 12:33:49 2020 @author: ppantina """ radName = input('Which radar? CRS, EXRAD, HIWRAP: ') flightDate = input('Which date?: yyyymmdd ') overwrite = input('Overwrite?: yes / no ') ##Import some libraries import h5py import glob import os import sys import numpy as np import L0_sub_parseCrsHkp import L0_sub_parseExradHkp import L0_sub_parseHiwrapHkp import L0_sub_parseAnt import L0_sub_parseIwg import L0_sub_parseInspva import L0_sub_readDigRxMixed_singleCh import L0_sub_unitsDescriptions import sub_params ##Choose the correct paths based on the working computer fileNameParams = 'radar_params_irma.xlsx' fileNameUnitDescriptions = 'radar_attrs.xlsx' ##Call subroutines for parameter/attributes data parsing hkpUnits, hkpDescriptions, \ antUnits, antDescriptions, \ gpsUnits, gpsDescriptions, \ navUnits, navDescriptions, \ radUnits, radDescriptions = L0_sub_unitsDescriptions.L0_sub_unitsDescriptions(fileNameUnitDescriptions, radName) Params = sub_params.sub_params (fileNameParams, flightDate, radName) ##Paths and info for raw radar/telemetry data fileNamesTele = Params ['path_HKP'] fileNamesIWG = Params ['path_IWG'] fileNamesRaw = np.sort(glob.glob(Params['path_RAW'])) if len(fileNamesRaw) == 0: print ('\nCheck your config file path names\n') sys.exit() ##Find hdf files that have already been written. We can skip these if desired (overwrite == no) ##Assuming we might have run the code previously, if we select [0:-1], we get all hdfs ##except the last one in the directory. The last hdf might not have been written fully, ##so we overwrite this one, but skip the rest. fileNamesOutExist= np.sort(glob.glob(Params['Directory_L0'] + '*' + radName +'*.hdf'))[0:-1] idx = [] for i, j in enumerate(fileNamesRaw): ##If the rawFile basename (which we use as the outFile basename) does NOT exist ##in the list of hdfs, note the idx number if os.path.basename(radName + '_' + fileNamesRaw[i].split('__')[1]) not in str(fileNamesOutExist): idx.append(i) #endfor i ##If we do NOT want to overwrite files, then use the idx to eliminate the ##already written hdfs from the list if overwrite == 'no': fileNamesRaw = fileNamesRaw[idx] print ('\nSkipping previously written files\n') ##Call subroutines for telemtry data parsing if 'EXRAD' in radName: antOut = L0_sub_parseAnt. L0_sub_parseAnt (fileNamesTele) #exrad only hkpOut = L0_sub_parseExradHkp. L0_sub_parseExradHkp (fileNamesTele) if radName == 'CRS': hkpOut = L0_sub_parseCrsHkp. L0_sub_parseCrsHkp (fileNamesTele) if 'HIWRAP' in radName: hkpOut = L0_sub_parseHiwrapHkp.L0_sub_parseHiwrapHkp(fileNamesTele) navOut = L0_sub_parseIwg. L0_sub_parseIwg (fileNamesIWG ) #all radars gpsOut = L0_sub_parseInspva. L0_sub_parseInspva (fileNamesTele, radName) #all radars ##Isolate time variable from each dict cpuHkp = hkpOut ['cpusec'] cpuNav = navOut ['cpusec'] cpuGps = gpsOut ['cpusec'] if 'EXRAD' in radName: cpuAnt = antOut ['cpusec'] ##Read gate/kind information from parameters file. Then convert to arrays ##Gates and kinds are needed for reading the raw radar data str_gates = Params['RangeGates' ].split(',') str_kind = Params['numProducts'].split(',') gates = np.zeros(8, dtype = 'int') #8 channels per radar. not all are used kind = np.zeros(8, dtype = 'int') #8 channels per radar. not all are used for i, j in enumerate(gates): if len(str_gates[i]) > 0: gates[i] = int(str_gates[i]) for i, j in enumerate(kind): if len(str_kind [i]) > 0: kind [i] = int(str_kind [i]) for a, b in enumerate(fileNamesRaw): #for all raw files: print ('\nStep ' + str(a) + ' of ' + str(len(fileNamesRaw))) ##If the file is NOT empty: if os.path.getsize(fileNamesRaw[a] )> 0: ##Create an HDF5 file. Use the input name for the output name fileout = radName + '_' + os.path.basename(fileNamesRaw[a]).split('__')[1] fileoutpath = Params['Directory_L0'] + fileout + '.hdf' if not(os.path.isdir(os.path.dirname(fileoutpath))): os.makedirs(os.path.dirname(fileoutpath)) f = h5py.File(fileoutpath, 'w') ##Read in the raw .dat RADAR filesC headers, dataout, extraPackets = L0_sub_readDigRxMixed_singleCh.L0_sub_readDigRxMixed_singleCh(fileNamesRaw[a], kind, gates) ##Isolate the time cpuData = headers['chan0']['cpusec'] ##Index telemetry data only where raw data is defined., plus/minus 20sec idxHkp = np.where((cpuHkp > cpuData[0]-20) & (cpuHkp < cpuData[-1]+20))[0] idxGps = np.where((cpuGps > cpuData[0]-20) & (cpuGps < cpuData[-1]+20))[0] idxNav = np.where((cpuNav > cpuData[0]-20) & (cpuNav < cpuData[-1]+20))[0] if 'EXRAD' in radName: idxAnt = np.where((cpuAnt > cpuData[0]) & (cpuAnt < cpuData[-1])) [0] ##Add global attributes to the file f.attrs['radName' ] = radName f.attrs['experimentName' ] = Params['experimentName' ] f.attrs['FlightDate' ] = Params['FlightDate' ] f.attrs['Description' ] = Params['Description' ] ##Add telemetry data if 'EXRAD' in radName: for i, j in enumerate(antOut.keys()): ##for ant data ##Create and populate a dataset for each key f.create_dataset('ant/' + j, data = antOut[j][idxAnt]) ##Add a description f['ant'].attrs['description'] = Params['EXRAD_ANT_Description'] ##Add Units/Descriptions based on attrs subroutine if j in antUnits.keys(): f['ant/' + j].attrs['units' ] = antUnits [j] else: f['ant/' + j].attrs['units' ] = '' if j in antDescriptions.keys(): f['ant/' + j].attrs['description'] = antDescriptions[j] else: f['ant/' + j].attrs['description'] = '' #endfor i #endif EXRAD for i, j in enumerate(hkpOut.keys()): ##for hkp data ##Create and populate a dataset for each key f.create_dataset('hkp/' + j, data = hkpOut[j][idxHkp]) ##Add a description f['hkp'].attrs['description'] = Params['HKP_Description'] ##Add Units/Descriptions based on attrs subroutine if j in hkpUnits.keys(): f['hkp/' + j].attrs['units' ] = hkpUnits [j] else: f['hkp/' + j].attrs['units' ] = '' if j in hkpDescriptions.keys(): f['hkp/' + j].attrs['description'] = hkpDescriptions[j] else: f['hkp/' + j].attrs['description'] = '' #endfor i for i, j in enumerate(gpsOut.keys()): ##for gps data ##Create and populate a dataset for each key f.create_dataset('gps/' + j, data = gpsOut[j][idxGps]) ##Add a description f['gps'].attrs['description'] = Params['GPS_Description'] ##Add Units/Descriptions based on attrs subroutine if j in gpsUnits.keys(): f['gps/' + j].attrs['units' ] = gpsUnits [j] else: f['gps/' + j].attrs['units' ] = '' if j in gpsDescriptions.keys(): f['gps/' + j].attrs['description'] = gpsDescriptions[j] else: f['gps/' + j].attrs['description'] = '' #endfor i for i, j in enumerate(navOut.keys()): ##for nav data ##Create and populate a dataset for each key f.create_dataset('nav/' + j, data = navOut[j][idxNav]) ##Add a description f['nav'].attrs['description'] = Params['NAV_Description'] ##Add Units/Descriptions based on attrs subroutine if j in navUnits.keys(): f['nav/' + j].attrs['units' ] = navUnits [j] else: f['nav/' + j].attrs['units' ] = '' if j in navDescriptions.keys(): f['nav/' + j].attrs['description'] = navDescriptions[j] else: f['nav/' + j].attrs['description'] = '' #endfor i ###################################### ##Create "data" group within the file ###################################### data = f.create_group('data') ##Add attributes to radar data group f['data'].attrs['LowPRF' ] = Params['LowPRF' ] + 's' f['data'].attrs['HighPRF' ] = Params['HighPRF' ] + 's' f['data'].attrs['AntennaSize' ] = str(Params['AntennaSize' ])+ 'm' f['data'].attrs['NumberOfChannels'] = Params['NumberOfChannels'] for i, j in enumerate(dataout.keys()): ##for raw data ##Create and populate a dataset for each key [ch] f.create_group('data/' + j) ##Add [ch] specific attributes f['data/' + j].attrs['Products' ] = Params['Products' + str(i)] f['data/' + j].attrs['Description' ] = Params['Description' + str(i)] f['data/' + j].attrs['PolarizationTransmit'] = Params['PolarizationTransmit'].split(',')[i] f['data/' + j].attrs['PolarizationReceive' ] = Params['PolarizationReceive' ].split(',')[i] f['data/' + j].attrs['PulseType' ] = Params['PulseType' ].split(',')[i] f['data/' + j].attrs['BandwidthReceiver' ] = str(eval(Params['BandwidthReceiver' ].split(',')[i])) + ' Hz' ##eval converts a string equation to number f['data/' + j].attrs['BandwidthWaveform' ] = str(eval(Params['BandwidthWaveform' ].split(',')[i])) + ' Hz' f['data/' + j].attrs['GateSpacings' ] = str(eval(Params['GateSpacings' ].split(',')[i])) + ' m' f['data/' + j].attrs['PulseWidth' ] = str(eval(Params['PulseWidth' ].split(',')[i])) + ' s' f['data/' + j].attrs['Frequency' ] = str(eval(Params['Frequency' ].split(',')[i])) + ' Hz' f['data/' + j].attrs['RangeGates' ] = eval(Params['RangeGates' ].split(',')[i]) for k, l in enumerate(dataout[j].keys()): ##for each key in [ch], add data ##Create and populate a dataset for each key f.create_dataset('data/' + j + '/' + l, data = dataout[j][l]) ##Add Units/Descriptions based on attrs subroutine if l in radUnits.keys(): f['data/' + j + '/' + l].attrs['units' ] = radUnits [l] else: f['data/' + j + '/' + l].attrs['units' ] = '' if l in radDescriptions.keys(): f['data/' + j + '/' + l].attrs['description'] = radDescriptions[l] else: f['data/' + j + '/' + l].attrs['description'] = '' #endfor k for k, l in enumerate(headers[j].keys()): ##for each key in [ch], add header info ##Create and populate a dataset for each key f.create_dataset('data/' + j + '/' + l, data = headers[j][l]) ##Add Units/Descriptions based on attrs subroutine if l in radUnits.keys(): f['data/' + j + '/' + l].attrs['units' ] = radUnits [l] else: f['data/' + j + '/' + l].attrs['units' ] = '' if l in radDescriptions.keys(): f['data/' + j + '/' + l].attrs['description'] = radDescriptions[l] else: f['data/' + j + '/' + l].attrs['description'] = '' #endfor k #endfor i f.close() #close the file #endif NOT empty #endfor a