o Z)d?d@Z*dkdAdBZ+dldCdDZ,dEdFZ-dGdHZ.dIdJZ/dKdLZ0dMdNZ1dOdPZ2dQdRZ3dSdTZ4ddVdWZ5dmdYdZZ6d[d\Z7d]d^Z8d_d`Z9dndadbZ:dodcddZ;dS)pN) coordinate)gdal) MiaplPyParser)readfileptimeutils) DSET_UNIT_DICTgeometryGEOMETRY_DSET_NAMESgiantIfgramStackgiantTimeseriesIFGRAM_DSET_NAMES ifgramStackTIMESERIES_DSET_NAMES timeseriesHDFEOSslcmiaplpy.load.slcFile unwrapPhasezmiaplpy.load.unwFile coherencezmiaplpy.load.corFileconnectComponentzmiaplpy.load.connCompFile wrapPhasezmiaplpy.load.intFileZionozmiaplpy.load.ionoFileheightmiaplpy.load.demFilelatitudemiaplpy.load.lookupYFile longitudemiaplpy.load.lookupXFile azimuthCoord rangeCoordincidenceAnglezmiaplpy.load.incAngleFile azimuthAnglezmiaplpy.load.azAngleFile shadowMaskzmiaplpy.load.shadowMaskFile waterMaskzmiaplpy.load.waterMaskFilebperpzmiaplpy.load.bperpFilec@sDeZdZddZddZddZddZd d Zd d Zd dZ dS) OutControlcCstj||_||_dSN)ospathabspathrun_filework_dir)selfZ batch_filer+r-K/home/exouser/operations/rsmas_insar/tools/MiaplPy/miaplpy/objects/utils.py__init__4s zOutControl.__init__cCs$||||dSr&)&remove_zero_size_or_length_error_filesraise_exception_if_job_exitedconcatenate_error_filesmove_out_job_files_to_stdoutr,r-r-r.clean8s zOutControl.cleancCs`t|jd}t|jd}t|jd}|||}t|dks.|D]}t|q&dS)N*.e*.job*.or)globr*lenr'remove)r, error_files job_filesout_fileZ list_filesitemr-r-r. remove_last_job_running_products>s   z+OutControl.remove_last_job_running_productscCsXt|jd}||D]}tj|dkrt|qt|dkr)t|qdS)ERemoves files with zero size or zero length (*.e files in run_files).r6rN)r9r*sortr'r(getsizer;file_len)r,r<r?r-r-r.r0Hs   z1OutControl.remove_zero_size_or_length_error_filesc Cs|t|jd}d}||D]+}t|}|}|D]}||vr+td||qWdn1s6wYqdS)rAr8zExited with exit codez ERROR: {0} exited; contains: {1}N)r9r*rBopen readlines Exceptionformat)r,files search_stringfilefrlinesliner-r-r.r1Ts z(OutControl.raise_exception_if_job_exitedc Cs:tj|jd|jddd}tj|rt|tj|jd|jddd}t |jd}t |dkst |d@}|D]5}| d| d |d | dt |}| | Wd n1srwYt|qGWd n1swYttj|tj|jd S) z Concatenate error files to one file (*.e files in run_files). :param directory: str :param out_name: str :return: None z/out_/z.er6rwz######################### z#### z N)r'r(r)r+r*splitisfiler;dirnamer9r:rEwritereadshutilmove)r,r>out_namer<outfilefnameinfiler-r-r.r2es&&  &       z"OutControl.concatenate_error_filescCst|jd}t|jd}tj|d}|dtj|j}tj|s/t|n t |t||D]}t ||q;|D]}t ||qFdS)z/move the error file into stdout_files directoryr7r8rz/stdout_N) r9r*r'r(rTbasenameisdirmkdirrWrmtreerX)r,r= stdout_filesZdir_nameZ out_folderr?r-r-r.r3s    z'OutControl.move_out_job_files_to_stdoutN) __name__ __module__ __qualname__r/r5r@r0r1r2r3r-r-r-r.r%3s   r%cs0eZdZd fdd ZddZd ddZZS) coord_revNcst||dSr&)superr/)r,metadata lookup_file __class__r-r.r/szcoord_rev.__init__cCsz t|jd|_Wnd|_Yd|jvr>d|_t|jd|_t|jd|_t|jd|_t|jd|_dSd|_|j rPt |j d d d |_ dSdS) N EARTH_RADIUSgMXAY_FIRSTTX_FIRSTY_STEPX_STEPFr.xml metafile_ext) float src_metadata earth_radiuskeysgeocodedlat0lon0lat_steplon_steprhread_attribute lut_metadatar4r-r-r.rEszcoord_rev.openTcCstd|jvr$tj|jdd|dd|_tj|jdd|dd|_nt|jd|_t|jd|_|j|jfS)Nrlrr) datasetName print_msgr)r}rvrrVrhlut_ylut_x read_image)r,rr-r-r.read_lookup_tables    zcoord_rev.read_lookup_tabler&)T)rbrcrdr/rEr __classcell__r-r-rir.resrecCs:t|d}t|WdS1swYdS)z(Calculate the number of lines in a file.rN)rEr:rF)r[rKr-r-r.rDs  $rDTcstjtj|\}}|}|dur|}n||}tj|s2d|}|dt7}t||dvrt |dfdd Dfdd Dgd }t fd d d Dred } nt fdd dDrsd} n|t fdd dDrd} nmt fdd dDrd} n^t fdd dDrd} nPdvrd} nIdvrd} nBt fdd dDrd} n4t fd d |Drt tt|@d!} ntd!krވd!} ntd!krd!} ntd"|| dkrt|} nD| dkrt|} n8tjd!krd#j vrtj} n"d$d%} ga| tr9d&dtD} tt| } ntd'|| D]\} }z |d(| | <WqB|| | <YqBt| | d)<d}|rr| vrr|}nd*d+}ga|trtd!}|durt|j| d,<d-| vr| d-| d.<d.| vrd/| d.<n|d0|d1|d2tj|d!d3|d4|d5g}d6d|D}t|d!krt d7|i} |!d8rt d9d |Drd:| d.<nKt d;d |Drd<| d.<d=d|D}t|d!kr| "t#$|d!n$t d>d |Dr*d?| d.<nt d@d |Dr?d.| vr?dA| d.<d.| vrJd/| d.<|d!}|dBvr`tj|\}}|dBvsS|se|}|!d0r| "t#%|d)| vr~|| d)<n|!d1r| "t#$|d)| vr|| d)<ny|!d2r| "t#&||| d)<nf|!d3r| "t#'|| d.d:krtj|}|(dCrdD| d)<n>|(dErdF| d)<n3|(dGrdH| d)<n(dI|vrdI| d)<n| dJ| d)<n|| d)<n|!d4r| "t#)||| d)<dKdLdMdNdO}| *d,dP}|dPkr3|| vr3||| d,<| d)+dQdR} | d krV|rQ|t, vrQt,|| dS<n/dT| dS<n*|rg|t, vrgt,|| dS<ndS| vr| t, vr|t,| | dS<ndU| dS<| d)+dQdR} | dkrdV| dS<n*dS| vr|r|t, vrt,|| dS<n| t, vrt,| | dS<ndU| dS<tj-|| dW<|rt#.| } | S)XaaRead attributes of input file into a dictionary Parameters: fname : str, path/name of data file datasetName : str, name of dataset of interest, for file with multiple datasets e.g. slc in slcStack.h5 date in slcStack.h5 height in geometryRadar.h5 latitude in geometryRadar.h5 ... standardize : bool, grab standardized metadata key name Returns: atr : dict, attributes dictionary Nzinput file not existed: {} zcurrent directory: z.h5z.he5rc g|] }t|tjr|qSr-) isinstanceh5pyGroup.0ifr-r.  z"read_attribute..cs.g|]}t|tjr|jdkr|qS))rrDatasetndimrrr-r.rs.)interferogramsrwrappedc3|]}|vVqdSr&r-rd1_listr-r. z!read_attribute..)r azimuthOffsetrc3rr&r-rrr-r.rr)rrrr c3|] }|vVqdSr&r-rrg1_listr-r.r)r displacementrc3rr&r-rrr-r.rrrrc3rr&r-rrr-r.rr)velocityrrreconsr c3rr&r-rrr-r.rr)igramfigramr c3rr&r-r)rr-r.rrrzunrecognized file type: WIDTHcSs8t|jdkrd|jvrtt|jdSdSdS)Nrr)r:attrsrvatr_listappenddictnameobjr-r-r.get_hdf5_attrs sz&read_attribute..get_hdf5_attrscSg|]}t|qSr-r:rr-r-r.rz!No attribute WIDTH found in file:utf8 FILE_TYPEcSs,t|tjr|jdkrt|dSdSdS)Nr)rrrrds_listrrr-r-r.get_hdf5_dataset.sz(read_attribute..get_hdf5_dataset DATA_TYPEINSAR_PROCESSOR PROCESSORmintpy.rscrp.par.hdr.vrtz.aux.xmlcSsg|] }tj|r|qSr-)r'r(rSrr-r-r.rMz(No metadata file found for data file: {}z.imgcs|]}|dVqdS)rNendswithrr-r-r.rSsnapcsr))rprrNrrr-r-r.rVriscecSsg|] }|dr|qS)rprrr-r-r.rXcsr)rNrrr-r-r.r\rgammacsr)rNrrr-r-r.r_rroipac)z.geoz.rdrz.fullunw.unw)cohcor.cor phase_ifg.intdemz file typeint8float32float64 complex64bytersdoublecfloatnone.UNITradian1r FILE_PATH)/r'r(splitextr]lowerrSrHgetcwdrGrFilervanylistsetr: ValueErrorr get_metadatar rrr visititemsnpargmaxitemsdecodestrrdtypecloseFileNotFoundErrorrupdater read_isce_xmlread_roipac_rscread_gamma_par read_envi_hdr startswith read_gdal_vrtgetreplacerr)standardize_metadata)r[r~Z standardizerrfbasefext test_filemsgpy2_mintpy_stack_fileskatrrnum_listkeyvaluedsr metafiles xml_filesmetafile dataTypeDict data_typer-)rrrr.r|sL                                              r|../defaults/miaplpyApp_auto.cfgcCstjtjt|}t|}|dur!|D]\}}|||<q|D]\}}|dkr9||vr9||||<q%ddg}|durCd} |ddkrm|ddkrm|d dks_|d |d<d } |d dksm|d |d<d } |D]}|d |dkr|d |dvr|d ||d |<d } qo|d|dks|ddkr|d|d<d } dd|Dddg}| d krC|d} t | d} t |dD]v} dd| ddD} | ds3t | dkr3| d}||vr3||r3||}|dkrd}n|dkrd}d|}t| dd d! d"d }|d#d$}td%|| d}| ||d} td&||||| | q| t| |dddddd'}|D]\}}||vr`||||<qO|S)(z7Replace auto value based on the input auto config file.Nautoz load.autoPathzload.compressionskipmiaplpy.subset.lalonomiaplpy.subset.yxmintpy.subset.lalorunmintpy.subset.yxmiaplpy.zmintpy.)rFmiaplpy.load.processorzmintpy.load.processorrcSsg|]}d|qS)rr-rrr-r-r.rrz-check_template_auto_value...tmprQrcSg|]}|qSr-striprr-r-r.rr=r%#rTyesFz=  rr *z\*z=[\s]* {}: {} --> {})r!TruerFalser)r'r(joinrT__file__r read_templaterrvrErrRrr:rrrefindallprintrHrUrrWrX) templateDictZmintpyTemplateDict auto_file templateFiletemplateAutoFiletemplateAutoDictrrZ common_keysstatustmp_filef_tmprNcZs_value new_value_str old_value_strZ specialValuesr-r-r.check_template_auto_values~                 r8c sd}t|tr|ddd}n t|tr|dd}fdddD}dd|Dd}t||d}t|d t|d }} |sKdd| |f}tjtj d  } | d vrht ||d } | |fSt ||d \} }| |fS)aRead one dataset and its attributes from input file. Parameters: fname : str, path of file to read datasetName : str or list of str, slice names box : 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate Returns: data : 2/3-D matrix in numpy.array format, return None if failed atr : dictionary, attributes of data, return None if failed Examples: from mintpy.utils import readfile data, atr = readfile.read('velocity.h5') data, atr = readfile.read('timeseries.h5') data, atr = readfile.read('timeseries.h5', datasetName='timeseries-20161020') data, atr = readfile.read('ifgramStack.h5', datasetName='unwrapPhase') data, atr = readfile.read('ifgramStack.h5', datasetName='unwrapPhase-20161020_20161026') data, atr = readfile.read('ifgramStack.h5', datasetName='coherence', box=(100,1100, 500, 2500)) data, atr = readfile.read('geometryRadar.h5', datasetName='height') data, atr = readfile.read('geometryRadar.h5', datasetName='bperp') data, atr = readfile.read('100120-110214.unw', box=(100,1100, 500, 2500)) Nr-csg|]}|qSr-r-rr[r-r.r-rzread..)rrpcSs*g|]}tj|rd|ddqS)rrP)r'r(existsrRrr-r-r.r.s*)r~rrLENGTHrrr)r~box) rrrRrr|intr'r(rr]rread_hdf5_fileread_binary_file) r[r=r~r dsname4atrrIZfext0rlengthwidthrdatar-r:r.rVs"   rVcsPt|}g}dd|DD] }||vr||q dd|Dfdd|Ds1|dgnttr9gtddDrKfddDdd dfd dD}t|d fd ddfD}fd d|D}t|dkr|d} nt|dkr|d} nt d || j dkr| |d|d|d|df} na| j dkr t j | jdt jd} |r|dgkrd| dd<nddfdd|DD} |D] } d| | | <q| | |d|d|d|df} t | } Wd| SWd| SWd| S1s!wY| S)a7 Parameters: fname : str, name of HDF5 file to read datasetName : str or list of str, dataset name in root level with/without date info 'timeseries' 'timeseries-20150215' 'unwrapPhase' 'unwrapPhase-20150215_20150227' 'HDFEOS/GRIDS/timeseries/observation/displacement' 'recons' 'recons-20150215' ['recons-20150215', 'recons-20150227', ...] '20150215' 'cmask' 'igram-20150215_20150227' ... box : 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate Returns: data : 2D/3D array atr : dict, metadata cSg|] }|ddqS)r9rrRrr-r-r.rYrz"read_hdf5_file..cSsg|]}d|vr|qS)r9r-rr-r-r.r\csg|]}|vr|qSr-r-r) ds_2d_listr-r.r]rGrcss|]}|VqdSr&)isdigitrr-r-r.rdrz!read_hdf5_file..csg|] }dd|qS)z{}-{}rrHr) ds_3d_listr-r.rerr9cs g|] }|dddqS)rr9)rrdsFamilyr-r.rhrrcg|] }|vr|qSr-rvrrr-r.rmrcs"g|] }dd|vr|qS)z/{}rrJr)r~r-r.rn"z%input dataset {} not found in file {}rr)rrTNcSrE)r9rrFrr-r-r.rrcsg|] }|r|qSr-)r)rjrLr-r.rr)get_slice_listrrrallrRrrr:rrHrrzerosshapebool_indexsqueeze)r[r~r= slice_listrr inputDateListdsNames dsNamesOldrrD slice_flag date_listdr-)r~rMrHrKrr.r?Bs^      & &    r?FcCstjtj|\}}|}t|dd}|d}t|d}t|d} |s.dd| |f}|dd} |d d } t|d d } |d d} d}d}|dvrddddd}| | vrg|| } |d dd}|dvrt d| }|r|dvrd}ni|dvrd}nb|dvr|r| drt d| }nQ|d vrt d| }|rd!|vrd}n=|r|d"krd}n2|d#krd$}n)| d%rd}n!|d&vrd}n|d'krd'}n| d(rd(}n| d)rd*}t || }n|d+vr>d} d } d} d} |d,vr d} d}n|d-vrd} n|d.vrd} d}n|d/vr#d0} n|d1vr+d2} n|d3vr=d} |r=| d4r=d}nm|d5krd} |d d6} d} |d7vrSnX|d-vr[d} nP|d8vrqd} |rn| d4rnd(}n=d'}n:|d9kr{d:} d}n0|d;vrd } n(|d| vr|d>d?krd } nt d@d|vr| |d<|r|St|||dA}||fS)Ba#Read data from binary file, such as .unw, .cor, etc. Parameters: fname : str, path/name of binary file datasetName : str, dataset name for file with multiple bands of data e.g.: incidenceAngle, azimuthAngle, rangeCoord, azimuthCoord, ... box : 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate Returns: data : 2D array in size of (length, width) in BYTE / int16 / float32 / complex64 / float64 etc. atr : dict, metadata of binary file rprqrr<rrrr BYTE_ORDERz little-endian number_bandsrschemeBILrphase)rrrrrrrr)rrr)band1 intensity magnituderrh)los)band2azhead)incLocallocalrjband3rQ)magamp)reanglerealimag)cpxcomplexrv)r)rrz.hgtz.msk)r).amp).demz.wgs84int16)z.flgz.bytrW).trans)rkazimuthrz big-endian)rrz.hgt_simrxrwz.ramp) .utm_to_rdcz.slc complex32)z.mlirBSQz byte order0zUnknown InSAR processor.)r=band)r'r(rr]rr|r>rupperrvrminrr,r)r[r~r=attributes_onlyrrr processorrBrCr  byte_ordernum_bandZband_interleavercpx_bandr rrDr-r-r.r@s                                 r@cs`tjtj|\}}|}t|}|d}|dvrt|dfddD}Wdn1s8wY|dkrT||vrTt |}|j dd |j a t S|d krgt |}|j dd |j a t S|d vr~||vr~t|}|j dd |j a t S|d vrt|}|j dd |j a t S|d vrt|}|j dd |j a t S|dvrt|}|j dd |j a t S|dvrt|}|j dd |j a t St|dt|dfdd}ga t|d|Wdt S1swYt S|dvr ddga t S|drddga t S|dddkr+d|vr+ddga t Sd ga t S)!z2Get list of 2D slice existed in file (for display)rrrcrr-)rrrrrr-r.r>rz"get_slice_list..NrFrr)r )r)r)r )r r<rcsZt|tjr)|jddfkr+|jdkrt|dStd |j|dSdSdS)Nrz#file has un-defined {}D dataset: {}) rrrrVrrZrwarningswarnrHr)rBrCr-r.get_hdf5_2d_datasetfs " z+get_slice_list..get_hdf5_2d_dataset)rzr|rrrir r!rbr2rrfrjr)r'r(rr]rr|rrrvrrE sliceListrZslcStackr rrr r r>rrr)r[rrrrrrrr-)rrBrCr.rS3s~ : 5 0 + & !      rSrcCs<ddlm}||}dd||D}|dkr|d}|S)zReturn the k most common item in the list L. Examples: 5, 8 = most_common([4,5,5,5,5,8,8,8,9], k=2) 'duck' = most_common(['goose','duck','duck','dog']) 'goose' = most_common(['goose','duck','duck','goose']) r)CountercSg|]}|dqS)rr-rr-r-r.rrzmost_common..r) collectionsr most_common)Lrrcntitem_mmr-r-r.rs rcCs|d}|d}tdtd|td||d}|dds)|d }n|}|d |d f}|dds.rrz -a zecho "z " | mail -s  zssh pegasus.ccs.miami.edu "cd z; "T)shellzError in email_miaplpy) subprocesssysr'getenvrr(r'r,Popenwaitexit)r+rrZ email_addressZtextStrcwdZflist file_listZ attachmentStrrfileListZmailCmdcommandr2r-rr. email_miaplpys*   rc sddlm} tjtjtd}tj|d}tj|s.td |t |||S||j ||j }t fdd|Drtdt ||||j }|D]\}}|vrk||krkd }qXd }qX|sxtd ||S|d } t| d } t|dD]S} dd| ddD} | dst| dkr| d}t| ddddd}|vrԈ||kr| ||d} td |||| | q| t | ||S)Nr)Templatez../defaults/miaplpyApp.cfgmiaplpyApp.cfgz/copy default template file {} to work directorycsg|]}|vqSr-rOrcdictr-r.r rGz/get_latest_template_miaplpy..zAobsolete default template detected, update to the latest version.TFz)No new option value found, skip updating rrQrcSrr-rrr-r-r.rrrrrr"rr r$)Zmiaplpy.objects.read_templaterr'r(r'rTr(rSr,rHrWcopy2optionsrrvrrErrRrr:rrrUrrX) r+rZlfilecfileldict orig_dictrrrr3r4rNr5r-rr.get_latest_template_miaplpysF   %        rcs@t|ddg}fdd|Dd}ddg}fdd|Dd}z4d d|d D}td d|dd D\}}td d|dd D\}}||||f} Wnd} Yz7dd|d D}tdd|dd D\} } tdd|dd D\} } | | | | f}W|| fSd}Y|| fS)zRead miaplpy.subset.lalo/yx option from template file into box type Return None if not specified. (Modified from mintpy.subsets) rrcg|]}|vr|qSr-r-rtmplr-r.r2rGz,read_subset_template2box..rrrcrr-r-rrr-r.r4rGcS$g|]}|ddddqS[r]rrrr-r-r.r7$,cSg|]}t|qSr-rsrrr-r-r.r8rG:cSrr-rrr-r-r.r9rGrNcSrrrrr-r-r.r@rcSrr-r>rrr-r-r.rArGcSrr-rrr-r-r.rBrG)rr)rRsorted) template_filervZkey_laloZkey_yxoptsrxlat1rylon1geo_boxy0y1x0x1pix_boxr-rr.read_subset_template2box+s*     rcsdddlm}ddlm}dd<dd<tdd\}}z#ttdddttdddg}d d |D}Wnd}Yz'fd d tDd}tt|dd}t | ddd d }Wnt }Yd} d| vrd} nd} |r| s|durd}t dt d|s|sddkrtd} || }t||d} |dur| |}| |}t d|t d|d} |durt |ddd } | sd| vr| |}t| |} t d| |dvrd|vrddt|dt|df}|D]}|vs |dvr&|||<q|d<| d<S)Nr)subsetr=rrrrprcSrE)rprrF)rxr-r-r.rZrz#read_subset_box..crNr-rOrinpsDictr-r.r` rrqrlTFzgWARNING: mintpy.subset.lalo is not supported if 1) no lookup file AND 2) radar/unkonwn coded datasetz ignore it and continue.rrr)rhz*input bounding box of interest in lalo: {}z#box to read for datasets in y/x: {}z/box to read for geocoded lookup file in y/x: {}NNonerr<)NNONE)Zmintpy.load_dataZ load_datarrrr9rdatasetName2templateKeyvaluesr|rRrrvr,ut get_file_listZ'update_box4files_with_inconsistent_sizerebbox_geo2radarcheck_box_within_data_coveragerHbbox_radar2georr>)rZmldrrrZ lookupFileZpathKeyrKrrwfnamescoordrZatrLutrr-rr.read_subset_boxJsl          rcCst|d }|}Wdn1swYdd|D}tt|t|}||}d}||krCtdtj|||fSt |dt |krUtd||fSt |}||fS)NrcSrE)r"rrF)rdater-r-r.rrz,update_or_skip_inversion..zSAll date exists in file {} with same size as required, no need to update inversion. zSNumber of new images is less than 10 --> wait until at least 10 images are acquired) rErFrrr,rHr'r(r]r:)Zinverted_date_listZ slc_datesrZinverted_datesZ new_slc_datesZ all_date_listZ updated_indexr-r-r.update_or_skip_inversions"  rcCsddlm}tj|d}tj|r2||}|jdd|}|}t |dt |d}nEd |}|d tj ||7}t | d d } | } t| } t| t| }|d } | d | d| d| d}t| } | }|||fS)Nr)rzinputs/slcStack.h5Frr<rz --template {}z --project_dir {} --work_dir {}Zload_slc)scriptr=rrQr)miaplpy.objects.slcStackrr'r(r'r;rE get_date_listrr>rHrTrrRparseread_inps2dictprepare_metadatar$read_inps_dict2slc_stack_dict_object)r+r/rslc_fileslcObjr_rg num_pixelsZscp_argsZParser_LoadSlcZ inps_loadSlcrr=stackObjr-r-r.read_initial_infos(      r csvddlm}ddlm}t|}d|d<tjtjt d}i}|j D]}t |}t ||d}||q$|D]\}} | ||<q;d|vrP|d|d <d fd d |D} | D]0}||} |d vrt||||<q_|dvrt||||<q_| r||||<q_d|vs|ddkrd|d<|dd|d<|dd|d<d|vrdd t|j D} || d|d<t|t|dd|d<|drtd|dtd|d|ddrtd|d|j|d|j|d}tj|d} |jd} tj| st| | tj|d}|jd}tj|s9t|||S)z.Read input Namespace object info into inpsDictr) auto_path)sensorNPLATFORMr )r.rr miaplpy.load.cs$g|]}|r|dqSr)rrRrprefixr-r.rrz"read_inps2dict..)rrrautoPath)rrrFrrrZ PROJECT_NAMEcSs g|] }tj|dkr|qS)r)r'r(r]rr-r-r.rrzSAR platform/sensor : {}z processor: {}rzAcheck auto path setting for Univ of Miami users for processor: {})rr+templatemiaplpy.load.metaFilez/inputs/referencemiaplpy.load.baselineDirz/inputs/baselines)Zmiaplpy.defaultsr mintpy.objectsr varsr'r(r'rTr(rrr)r8rrrvr>rrproject_name2sensor_namerr,rHZ get_auto_pathr+r;rWcopytreer))inpsr r rZ auto_templaterr[temprrkey_listrZ reference_dirZ out_reference baseline_dirZ out_baseliner-rr.rsj                rc CsRddl}|d}d|}tdtd||dvrHdd|DD] }ttt||dkrEd |||}t|t|q%dS|d krtj tj |d }tj |d }t t|d }t|d kry| d|d z'|d} |d} tj |d} dj|||| | | d}t|t|WdSYdSdS)Nrrzprep_slc_{}.pyrz&prepare metadata files for {} products)rrrcSs$g|]}|dr|dr|qS)rr)rrrr-r-r.rrz$prepare_metadata..z{} {}rrrrz No input metadata file found: {}rrz.{s} -s {i} -f {f} -m {m} -b {b} -g {g} --force)srrmbg)rrHr,rvr:r9rr'systemr(rTr]rr) rrr script_namercmdZslc_dirrZ meta_filesZ meta_filerZgeom_dirr-r-r.rsH    rrcCsddlm}ddl}|dkrXt|t|dtj}|j}|j} |t |} | t |} tj d| | d} tj ||| dd}t|tj}tj |d|tj dd dd}tStd || } | |d |}|||||| |||tS) Nr)LooksrrZENVI)rHrCr)rZVRTrJzMultilooking {0} ...rp)Zmroipac.looks.Looksr&isceobjr,rrr RasterXSize RasterYSizer>TranslateOptions TranslaterH createImageloadZ setDownLooksZsetAcrossLooksZ setInputImageZsetOutputFilenameZlooksZoutfilie)r\rZrlksZalksZmultilook_toolr&r'rZxSizeZySizeZoutXSizeZoutYSizeZgdalTranslateOptsZinimgZlkObjr-r-r. multilook<s4       r/皙?c Cs||t||}tddd}|t|ddt|}tt|g}|D]3}tjdd}dtd|dtd |d|d} tj tj | d d dd } | |||k<q+|||k} t | S) Ng{Gz?rgMbP?gQ?g)\(?errPrr)initial) rsrarangesqrtrUr:ogridsumexpaminamaxr) N1N2alphaN distanceslamdaZalpha_crnpvaluecritical_distancer-r-r.ks_lutds 0  rCcCs0t|}t|||jd}t|}|S)z. Estimate Correlation matrix from an ensemble.r)rmatrixmatmulgetHrVcov2corr)ZCCGsamZCCGScov_mat corr_matrixr-r-r.est_corrss rJc Cs>tttt|}t||}t|t|}|S)z= Converts covariance matrix to correlation/coherence matrix. )LApinvrdiagflatr4diagrE transpose)Z cov_matrixDyrIr-r-r.rGs rGcstdtdtdddlm}m}m}tddttD}i}dd|DD]/}t|}||vr[t t t ||d }t |dkr[|||<td j ||||d q,d } | |vrmtd | dSt||d| d}i} |D]}t ||} | | |<tdj ||| dq|t| tfddDrd} | d7} t| |ddvrtj|dd} nd} |ddvrtj|dd}nd}t|}i}|| D]}tt|d ddddtjd}d}| dur| |krd}|dur||krd}|rai}tt |D]:}||}||d}|vr3|||<qfd d||D}t |dkrM|d||<qtd! |q||d"}||<qt |dkrp||d#}|Sd}|S)$z5Read input arguments into dict of slcStackDict objectrzsearching slcs infozinput data files:r)slcDatasetNames slcStackDictslcDictcSrr-rrr-r-r.rrz8read_inps_dict2slc_stack_dict_object..cSsg|] }|tvr|qSr-)rrvrr-r-r.rrrpz{:<{width}}: {path})rCr(rz(WARNING: No reqired {} data files found!Nr=)rdsNameznumber of {:<{width}}: {num})rCnumc3s|] }|dkVqdS)rNr-r) dsNumListr-r.rrz7read_inps_dict2slc_stack_dict_object..z@WARNING: NOT all types of dataset have the same number of files.z8 -> skip interferograms with missing files and continue.zmiaplpy.load.startDaterz%Y%m%dzmiaplpy.load.endDaterrqDATETFcsg|]}|vr|qSr-r-rdatesr-r.rrGz%WARNING: {} file missing for image {})rZ datasetDict) pairsDict)r,rrRrSrTmaxrrrvrr9rr:rH!skip_files_with_inconsistent_sizerrrstrptimeryyyymmddr|rRrange)rrRrSrTmaxDigit dsPathDictrUrrIZdsName0Z dsNumDictZnum_filer start_dateend_dateZ dsNameListr\ZdsPathZdate_valZ include_dateZ slcPathDictrZdsPath1ZdsPath2rr r-)rZrWr.rs                rcsdd||D}dd|D}dd|D}d}tdd||fDrA|dur+d }n|d d \}}|t|ks?|t|krAd }|rt|} t|} d } | d 7} | d| | 7} | d7} | d7} t|} dd|D} tt| D]C}||| ks||| kr| |}t || D]}fdd||D}t|dkr|| |dq| d|||||7} qv| d7} | d7} | d7} | d7} t | |S)z?Skip files by removing the file path from the input dsPathDict.cSs"g|] }t|ddddqS)rprrq)r|rR)rr[r-r-r.rrPz5skip_files_with_inconsistent_size..cSg|]}t|dqS)r<r>rrr-r-r.rrGcSrf)rrgrhr-r-r.rrGFcss |] }tt|dkVqdS)rN)r:r)r size_listr-r-r.rz4skip_files_with_inconsistent_size..NTrzQ ********************************************************************************zQ WARNING: NOT all input unwrapped interferograms have the same row/column number!z" The most common size is: ({}, {})z ------------------------------z) The following dates have different size:cSr)rXr-rhr-r-r.rrcs&g|]tfddDrqS)c3s |] }|ddvVqdS)rNr-)rr`rr-r.rrjz?skip_files_with_inconsistent_size...)rT)rrYrmr.rs rz {} ({}, {})z' Skip loading the interferograms above.z* Continue to load the rest interferograms.zR ******************************************************************************** ) rrrrrHrrvrar:rr`r;r,)rcrrUr length_list width_listZdrop_inconsistent_filesZ max_box_widthZmax_box_lengthZ common_lengthZ common_widthrr\r_rrrr-rYr.r^sJ     r^c Cs|rtndd}|d|rdd|D}n.|rBt|d} dd| jD}Wdn1s5wY|d |ntd |dur|rRtj |sVt d |d |i}tj |d } t |dt |df} | dvrt|dS} t | jdt | jdf} | D]8} | | }t|tjr|jdd| kr|j|j|ddg|| <qt|j}| |dd<|jt|dg|| <qWdn1swYnt|}|d}|D] }|t| dg||<qtjtj|}tj|s t||d|t|d}|d|tdd| D}| D]q} || d}|| d }|}| dvrAd}t|dkrRd|d |df}n|}|dj| |t|t||d| | vrw|j| |||d|d }n|| }t|| dkr|| ddurt|| d|dd<q*|D] \} }t||j| <q|dur|D]\} }|dur||| jd!<|d"| d#|d$|qWdn 1swY|d%||S)&a:Create HDF5 file with defined metadata and (empty) dataset structure Parameters: fname - str, HDF5 file path ds_name_dict - dict, dataset structure definition {dname : [dtype, dshape], dname : [dtype, dshape, None], dname : [dtype, dshape, 1/2/3/4D np.ndarray], #for aux data ... } metadata - dict, metadata ds_unit_dict - dict, dataset unit definition {dname : dunit, dname : dunit, ... } ref_file - str, reference file for the data structure compression - str, HDF5 compression type Returns: fname - str, HDF5 file path Example: layout_hdf5('timeseries_ERA5.h5', ref_file='timeseries.h5') layout_hdf5('timeseries_ERA5.5h', ds_name_dict, metadata) # structure for ifgramStack ds_name_dict = { "date" : [np.dtype('S8'), (num_ifgram, 2)], "dropIfgram" : [np.bool_, (num_ifgram,)], "bperp" : [np.float32, (num_ifgram,)], "unwrapPhase" : [np.float32, (num_ifgram, length, width)], "coherence" : [np.float32, (num_ifgram, length, width)], "connectComponent" : [np.int16, (num_ifgram, length, width)], } # structure for geometry ds_name_dict = { "height" : [np.float32, (length, width), None], "incidenceAngle" : [np.float32, (length, width), None], "slantRangeDistance" : [np.float32, (length, width), None], } # structure for timeseries dates = np.array(date_list, np.string_) ds_name_dict = { "date" : [np.dtype("S8"), (num_date,), dates], "bperp" : [np.float32, (num_date,), pbase], "timeseries" : [np.float32, (num_date, length, width)], } c_sdSr&r-)argskwargsr-r-r.Ssz#write_layout_hdf5..rcSi|]\}}||qSr-r-rrrr-r-r. Xz%write_layout_hdf5..rcSrsr-r-rtr-r-r.ru[rvNzgrab metadata from ref_file: {}zNo metadata or ref_file found.z"No ds_name_dict or ref_file found!z(grab dataset structure from ref_file: {}rr<rrrrzcrerate directory: {}az create HDF5 file: {} with w modecSrr-rrr-r-r.rrz%write_layout_hdf5..r)rlzfrQrzOcreate dataset : {d:<{w}} of {t:<25} in size of {s:<20} with compression = {c})r`rQtrr5T)rVmaxshaperchunksrrzadd /rvrrrVrrtuplerrSrTr)r^makedirsr]r:rcreate_datasetrarray)r[ ds_name_dictrg ds_unit_dictref_filerrvprintmetarLrshape2d shape2d_origrrds_shapeds_namesds_dtypeds_namefdirr max_digitr  data_shapeds_comp max_shaperr-r-r.write_layout_hdf5#s0          $  4r)NTN)Nr N)NNT)NN)NNFr)Nr)rr)r0)Nr)NNNNNT)s   0     e( } O / K' N  %2SG % ( f2