ZfdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z m Z m Z mZddlmZmZmZddlddZddZddZdd Z dd Zdd Zdd ZddZddZddZed fdZd dZ d!dZ!d"dZ"d#dZ# d$dZ$y)%z.Miscellaneous utilities - dependent on utils0.N)GEOMETRY_DSET_NAMESderamp ifgramStack timeseries)ptimereadfile writefile)*c4tj|}|ddt|dt|df}t|d|dzdz }t|d|dzdz }d|jvrLt |d}t |d}t |d }t |d }|||zz} |||zz} | | fS|||dz|dzf} t tj |d | d} t tj |d | d} | | fS)z#Get the lat/lon of the scene centerrWIDTHLENGTHY_FIRSTX_FIRSTY_STEPX_STEPlatitude datasetNamebox longitude)rread_attributeintkeysfloatread) geom_filermetacol_crow_clat0lon0lat_steplon_steplat_clon_cbox_cs L/home/exouser/operations/rsmas_insar/tools/MintPy/src/mintpy/utils/utils1.pyget_center_lat_lonr+s9  " "9 -D {!Sg'T(^)<= Q#a&A% &E Q#a&A% &EDIIKT)_%T)_%h(h(x%''x%'' %<uQwa0hmmI:5QRSTUhmmI;ERSTUV %<c|r|dk(rtd|}n%tjj|dd}tjj|ddz}t |||gddk(r}t || dk(rCtjj |sd |z}|d z }t |t|||| }td |zt|j||}td|ztj|tjt}|dddfjtjj!}t#|dddf}|||fS)aYCalculate deramped standard deviation in space for each epoch of input timeseries file. Parameters: timeseries_resid_file - string, timeseries HDF5 file, e.g. timeseries_ERA5_demErrInvResid.h5 mask_file - string, mask file, e.g. maskTempCoh.h5 ramp_type - string, ramp type, e.g. linear, quadratic, no for do not remove ramp Returns: std_list - list of float, standard deviation of deramped input timeseries file date_list - list of string in YYYYMMDD format, corresponding dates std_file - string, text file with std and date info. Example: import mintpy.utils.utils as ut std_list, date_list = ut.get_residual_std('timeseries_ERA5_demErrInvResid.h5', 'maskTempCoh.h5')[:2] noNo ramp removalr_ramp.h5z_std.txtFout_filein_filereadablerunr2r3-Can not find input timeseries residual file: $ Re-run dem_error.py to generate it. ramp_type mask_filer2zBcalculating residual standard deviation for each epoch from file: maskFileoutFilezread timeseries RMS from file: dtypeNr)printospathsplitext run_or_skipisfile Exception run_deramprtimeseries_stdnploadtxtbytesastypestrfloat32tolistlist) timeseries_resid_filer;r: deramped_filestd_filemsgfcstd_list date_lists r*get_residual_stdrY2si  T)  - 77++,AB1EFhO ww .q1*>"78EF[[>>n$!+)''* ! RS``am,;;YX`;a +H 45 HE * 1 1# 6B!Q$xrzz*113HR1XI Y ((r,c|r|dk(rtd|}n%tjj|dd}tjj tjj |}tjjtjj |d}tjj|d|d}t|||gdd k(r}t|| d k(rCtjj|sd |z}|d z }t|t|||| }td|zt|j||}td|ztj|t j#t$}|dddfj#tj&j)} t+|dddf} | | |fS)aVCalculate deramped Root Mean Square in space for each epoch of input timeseries file. Parameters: timeseries_resid_file : string, timeseries HDF5 file, e.g. timeseries_ERA5_demErrInvResid.h5 mask_file : string, mask file, e.g. maskTempCoh.h5 ramp_type : string, ramp type, e.g. linear, quadratic, no for do not remove ramp Returns: rms_list : list of float, Root Mean Square of deramped input timeseries file date_list : list of string in YYYYMMDD format, corresponding dates rms_file : string, text file with rms and date info. Example: import mintpy.utils.utils as ut rms_list, date_list = ut.get_residual_rms('timeseriesResidual.h5', 'maskTempCoh.h5') r.r/rr0rms_.txtFr1r5r6r7r8r9z4 calculating residual RMS for each epoch from file: r<z(read timeseries residual RMS from file: r?Nr)rArBrCrDdirnameabspathbasenamejoinrErFrGrHrtimeseries_rmsrJrKrLrMrNrOrPrQ) rRr;r:rSfdirfbaserms_filerUrVrms_listrXs r*get_residual_rmsrfbs&  T)  - 77++,AB1EFhO 77??277??=9 :D GG  RWW--m< =a @Eww||DDt"45HH/Di.P[`aejj  7L MQV V77>>"78EF[[>>n$!+)''* !  EmSTm,;;<   4X => HE * 1 1# 6B!Q$xrzz*113HR1XI Y ((r,ctj|}|d}|dk(rt|j|}nt d|zyd|d<t j ||||S)zDGenerate mask file for non-zero value of input multi-group hdf5 file FILE_TYPErrz5Only ifgramStack file is supported for now, input is Nmaskr2metadata)rrr nonzero_maskrAr write)Filer2ratrkrjs r*rmrmsl  ! !$ 'C KAM4 --+-F EaGHC  OOD8c: Or,Fc  d}tj|} | d} |sddt| dt| df}| dk(r|n>tjj tjj |d} |dnd } | |rd nd z } | | zd z} |j| rtd ||\}}||fSdtjj |d}d|d}d|d}d|d} t| }|j}|j|r |Dcgc] }d|vs| c}d}n|} |Dcgc] }d|vs| c}d}t| |gd}| dk(retj|d5}t||j j#dtjj%|}dddntjj%|}tjj%| }||k(r,||k(r'|dk(r"|krt| dz|| \}}||fS|rd|vrd}nd}| dk(r`t'|}|jd|j)||||||\}}|j*}|j,} |jn| dk(r"t/|j)|||| \}}ntj0||!d}!|rgtjj3|rHtd"|ztj0|d#|$d}"t4j6|!|"t|k(<|d%|!|!|kD<d|!|!|k<t5j8|!}tjj |g}|rtd&| zt| d'}|j;||z|z|zt=|}#| dk(rG|j;d(t?|#D]'}|j;d)|||| |||fz)n?|j;d*t?|#D] }|j;||d+||d,d"|j|| \}}t=|d%k(r |d}|d}||fScc}w#d }YqxYwcc}w#d }YgxYw#1swYxYw#YxYw)-aRead/Calculate Spatial Average of input file. If input file is text file, read it directly; If input file is data matrix file: If corresponding text file exists with the same mask file/AOI info, read it directly; Otherwise, calculate it from data file. Only non-nan pixel is considered. Parameters: fname - string, path of input file maskFile - string, path of mask file, e.g. maskTempCoh.h5 box - 4-tuple defining the left, upper, right, and lower pixel coordinate saveList - bool, save (list of) mean value into text file reverseMask - bool, perform analysis within masked regions instead of outside of them threshold - float, calculate area ratio above threshold instead of spatial average Returns: meanList - list(float) or float, average value in space for each epoch of input file dateList - list(str) or str, for date info date12_list, e.g. 101120-110220, for interferograms/coherence date8_list, e.g. 20101120, for timeseries file name, e.g. velocity.h5, for all the other file types Example: meanList = spatial_average('inputs/ifgramStack.h5')[0] meanList, date12_list = spatial_average('inputs/ifgramStack.h5', maskFile='maskTempCoh.h5', saveList=True) ctj|tjt}|dddfDcgc] }t |}}|dddfDcgc]}|}}||fScc}wcc}w)Nr?rr)rJrKrLrMrNr)fname txtContentimeanListdateLists r*read_text_filez'spatial_average..read_text_fileslZZU3::3? &0A&67E!H77)!Q$/0!A00!!80s A( A-rhrr r rN SpatialAvg AreaRatioRevMskr\z;Input file is spatial average txt already, read it directlyz # Data file:  z # Mask file: z # AOI box: z # Threshold: z # AOI box:z # Mask file:Fr1rMODIFICATION_TIMEskipz! already exists, read it directlyoffsetT print_msg)rr=r useMedian reverseMask thresholdr)r=rrr)rzmask from file: rjrrz-write average value in space into text file: wz'# DATE12 Mean Btemp/days Bperp/m Num z%s %.4f %8.0f %8.1f %d z# DATE12 Mean  z.4f) rrrrBrCrDr_endswithrAopen readlinescloserEh5pyrorattrsgetgetmtimerspatial_average pbaseIfgram tbaseIfgramrrrFrJnannanmeanrnlenrange)$rtrr=rsaveListcheckAoirrryrprqprefixsuffixtxtFilerwrx file_line mask_lineaoi_line thres_linefllinesrv aoi_line_origmask_line_origupdate_mask_fileftitorobjpbasetbasedatarjnumLines$ r*rrs84"  ! !% (C KA !SW&CM(:;.[BGG4D4DRWWEUEUV[E\4]^_4`F&.\KF +h2-Fvo&G ~~f KL+E2(!!!!1!1%!8 9//334GIYIYZ_I`ab c c!!%(B WW  g & X %"i/$."H '== >!/!8 HhX% % x;.   M%  5! 00## 1 ( l '.>># ? (}}U,Q/ x0 $X- .==v3GJD-/VVD[)) *  %&D ! "&'D" #::d#GG$$U+, =gEF '3  9$X-j89h-  HHF G7^ L7"1+x{E!HeAhJKL L HH* +7^ @HQK=8A;s*;2>? @  (0Hh 8}A;A; X O!H # " G N  c c  s<-S*R/. R*8R*<R/SR> R9R9R>*SASA7S*R//R63S9R>>SSS SSc2tj||}|d}|dvr+td|dtj|d}||fS|tj j |d}|st|dk(r|d k(rd }ngd |vrd }n`d |d}nY|dk(rN||vrPtj j|jddj|d}d|d}nd |d}|r;tj j|rtj|d} | |fS|dk(r0t|j|} d |vr d|d<d|d<n)||d<n#|dk(rt|j} d|d<|rtj || |fS)aCalculate temporal average of multi-temporal dataset, equivalent to stacking For ifgramStack/unwrapPhase, return average phase velocity Parameters: fname - str, file to be averaged in time datasetName - str, dataset to be read from input file, for multiple datasets file - ifgramStack - only e.g.: coherence, unwrapPhase updateMode - bool outFile - str, output filename None for auto output filename False for do not save as output file Returns: dataMean - 2D np.ndarray outFile - str, output file name Examples: avgPhaseVel = ut.temporal_average('ifgramStack.h5', datasetName='unwrapPhase')[0] ut.temporal_average('ifgramStack.h5', datasetName='coherence', outFile='avgSpatialCoh.h5', updateMode=True) rirh)rrz0WARNING: input file is not multi-temporal file: z, return itself.rrr coherencezavgSpatialCoh.h5 unwrapPhasezavgPhaseVelocity.h5avg.h5ravgDisplacementvelocityzm/yearUNIT displacementrk)rrrArrBrCrDr_splitrFrtemporal_averagerr rn) rtr updateModer>rprqrext processMarkdataMeans r*rrUs$  ! !%[ AC KA-- @GWXY}}U#A&U{ggu%a(M!+-0G"k13G #K=4Gl":"$''"2"25"9"?"? "Ma"P"V"VWZ"[\]"^K / }C@Gwc*bggnnW-==)!,   Mu%66;6O K ')C "CK*C  l e$557)K7SA W r,c l|sgSt|tr|g}|Dcgc]}|| }}g}|D]C}tj|}|tt t |t |z z }E|r,|Dcgc]!}t jj|#}}|t |D]x}tj|}|dvr%d|jvs/|j|A|dvr%d|jvsX|j|jd|} t| |Scc}wcc}w)aRGet all existed files matching the input list of file pattern Parameters: file_list - string or list of string, input file/directory pattern abspath - bool, return absolute path or not coord - string, return files with specific coordinate type: geo or radar if none, skip the checking and return all files Returns: file_list_out - list of string, existed file path/name, [] if not existed Example: file_list = get_file_list(['*velocity*.h5','timeseries*.h5']) file_list = get_file_list('timeseries*.h5') )geor)radarrdrrdcz un-recognized input coord type: ) isinstancerNglobsortedrQsetrBrCr^rrrremove ValueError) file_listr^coordx file_list_outrtfnamesrvrprUs r* get_file_listrs=  )S!K &7q7I7MH5!S[3}3E%E FGG H5BC+C C -( &E))%0CCHHJ.!((/11 *!((/8@ o% & +8DsD,D,:&D1c |sBgd}gd}g}|D]3}||Dcgc]"}tjj||$c}z }5g} t|}d}|D]:}t j |dD]} t j|| d|}:<|s|r t d y|rtjj|}|Scc}w#|r!t dt dt |YyxYw#YxYw) aFind lookup table file with/without input file pattern Parameters: filePattern - list of str abspath - bool, return absolute path or not print_msg - bool, printout message or not Returns: outFile - str, path of the lookup file )zgeometryRadar.h5zgeometryGeo_tight.h5zgeometryGeo.h5zgeomap*lks_tight.transzgeomap*lks.transzsim*_tight.UTM_TO_RDCzsim*.UTM_TO_RDC)inputsr}z ../inputsz-ERROR: No geometry / lookup table file found!zIt should be like:N)r rangeCoordF)rrz9No lookup table (longitude or rangeCoord) found in files.) rBrCr`rrArrrr^) filePatternr^rfileListdirListr]rt existFilesr>dsNames r*get_lookup_filers @.  PG hOUBGGLL%8O OK PJ";/ G&1 F  e5I     M N''//'* NAP   A B & ' +  s'C C 5C6 %C36C:cB t|tr|g}|D]}|tvs td||st j }dDcgc]"}tj j||$}}t||}t|dk(r|r tdyt|D]( t fd|Ds|j *t|dk(r|rtd|d y|d}|rtj j|}|Scc}w) z4Find geometry file containing input specific datasetz$unrecognized geometry dataset name: )z *geometry*.h5z*/*geometry*.h5z../*/geometry*.h5)rrzNo geometry file found.Nc3LK|]}|tjvywN)rget_dataset_list).0dsetrts r* z$get_geometry_file.. s!Rt844U;;Rs!$zNo geometry file with dataset z found)rrNrrrBgetcwdrCr`rrrArQanyrr^) dset_listwork_dirrr^rrrv fname_listrrts @r*get_geometry_filers)S!K L * *CD6JK KL 99;6oo"'',,x+oJoz7J :!  + ,j!% R R R   e $% :!  29+VD E1 IGGOOI. 'ps 'Dc d}tj|}|jD]#\}}||jvs|||k7s"d}%|st d|z|S|dz}t |d}t |D]E} | j j|dD cgc]} | j } } | jdst| dkDr| d}tj| dd d jd dj }||jvr|||k7r~|} d D]} | j| d | } |dz| z} tj| | d}|j|||}| j||d} t d|d|d|||j| H|jt!j"|||Scc} w)zEUpdate option value in template_file with value from input extra_dictFTz)No new option value found, skip updating z.tmprr)%#rr~r}r)r []()\z[\s]* z: z --> )r read_templateitemsrrArstripr startswithrrNreplacerefindallrnrshutilmove) template_file extra_dict delimiterupdate orig_dictkeyvaluetmp_filef_tmplinervc value2searchsymbol old_value_str new_value_strs r*update_template_filersF&&}5Ioo' U *//# # 35(@F  9-GHV#H 3 E]# $ 2 29a @ A1QWWY A Az*s1vzA$CKK!dB/55c:1=CCEEjoo''JsOu,D % 7OF#/#7#71VH #NLO )1,> " < >q A - 5 5eZ_ M ||M=!DSEE7% 3/@AB D/0 KKM KK-( 7 Bs"G-c l|rtnd}tj|}t|j }t ||}|s |d|St jj|d}|dvrtj|d5}t|jD]g\} } | dk(s| ,| |vs|jj| |d| 9t| |j| <|d | d t| i ddd|St|jD]S\} } | dk(s| "| |vs|j| |d| /t| || <|d | d t| Ut!j"||d z| |S#1swY|SxYw) a8Add/update input attribute of the give file. Parameters: fname - string, path/name of file atr_new - dict, attributes to be added/updated if value is None, delete the item from input file attributes Returns: fname - string, path/name of updated file cyr)argskwargss r*zadd_attribute..Msr,zeAll updated (removed) attributes already exists (do not exists) and have the same value, skip update.r)rz.he5zr+NoneNremove z add/update z = z.rscr)rArrrQrupdate_attribute_or_notrBrCrDrroiterrrpoprNr write_roipac_rsc) rtatr_newrvprintrpkey_listrfextrrrs r* add_attributerEs U%AF  ! !% (CCHHJH%Wc 2F 8 9  77  E "1 %D  YYud # ?q"7==?3 ? UF?emh C(/$'u:AGGCL[SU => ? ?: Lw}}/ ;JC%-(?GGCLWSE?+u:CSESZL9: ; ""3f J L; ?: Ls+F):AF))F3c|s|ddfSg}g}|D]?}tj|}|j|d|j|dA|r|n t|}|r|n t|}t |}|j |t |k7s|j |t |k7rtdtdtdtd|d|td t|||D]5\}}} | |k7s||k7st|d | d ||j|7td tt |ztd |||fS)zXCheck file size in the list of files, and drop those not in the same size with majority.Nr r z-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%z2WARNING: Some files may have the wrong dimensions!z$All files should have the same size.z3The width and length of the majority of files are: z, zNBut the following files have different dimensions and thus will not be loaded:z width: z length: z Number of files left: z,%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%) rrappend most_commonrQcountrrAziprrN) r mode_width mode_length width_list length_listrtrpfname_list_outlengthwidths r*check_file_sizer$}sw 4%%JK*%%e,#g,'3x=)* *{:/FJ!,++k2JK*%N$J7  -[1AA => BC 45 ;( ) ^_$' K$L - E65 "f &;{5'F8DE%%e, - (S-@)AAB <= :{ 22r,cf t||d}tj||S#d}Y|SxYw)aCheck if any file in the file list 1) exists and 2) readable Parameters: file_list : str or list(str), file name with/without wildcards abspath : bool, return absolute file name/path or not Returns: file_path : string, found file name/path; None if not. )r^rN)rrr)rr^files r* is_file_existr's> Y8;% K Ks%)0c|syt|tr|g}td|Dsy|r tj|dd|rJt|}|r=td|D}td|D}||kDry|rt |d|d y #|rt |ddt j|dYyxYw) aCheck whether to update out_file or not. return run if any of the following meets: 1. out_file is empty, e.g. None, [] 2. out_file is not existed 3. out_file is not readable by readfile.read_attribute() when readable=True 4. out_file is older than in_file, if in_file is not None Otherwise, return skip. If in_file=None and out_file exists and readable, return skip Parameters: out_file - string or list of string, output file(s) in_file - string or list of string, input file(s) readable - bool, check if the 1st output file has attribute 'WIDTH' print_msg - bool, print message Returns: run/skip - str, whether to update output file or not Example: if ut.run_or_skip(out_file='timeseries_ERA5_demErr.h5', in_file='timeseries_ERA5.h5'): if ut.run_or_skip(out_file='exclude_date.txt', in_file=['timeseries_ERA5_demErrInvResid.h5', 'maskTempCoh.h5', 'smallbaselineApp.cfg'], readable=False): r5c3ZK|]#}tjj|%ywr)rBrCrFrrvs r*rzrun_or_skip..s7277>>!$7)+rr z% exists, but can not read, remove it.c3ZK|]#}tjj|%ywrrBrCrr*s r*rzrun_or_skip..s=((+=r+c3ZK|]#}tjj|%ywrr-r*s r*rzrun_or_skip..s>((+>r+z exists and is newer than z --> skip.r) rrNallrrrArBrrmaxmin)r2r3r4rt_int_outs r*rErEs0  h $ zH7h77   # #HQK 0 9( =W==D>X>>Ee| "?+K{F399;G&";/'7;34#((*6 U F?s&6&;&;&== 0 5L 6  N#((*6 U  N'') ) .u 5L 6 r,c  tj}tjj|}tjj tjj |\} } t j|} | d} t| d} t| d}td|d||r|n$tjj|| d| }| dk(r|}tjj|r't j|d}td |zn2tj| |ftj }td d }|rxtjj|d | d}t!|d5}|j#d|d|j#d|d|j#d|dd d d | dk(r,t%j&||dtdt)|j+}t-|}t/j0|}t3|D]}|r,t!|d5}|j#||dd d d t j|||d}t5|||| |d}t%j6||d||dzd| d|gd|j9|dz|dzd |!|j;td"|n8| dk(rt=|}|j!d#|sd$}t?j@|d5}||}|d}||jCvr||}td%|n?|jE||jF| |ftjHdd &}td'|t/j0|jF}t3|jFD]}|r?t!|d5}|j#tK|jL|dd d d ||d d d d f}t5|||| |d}|||d d d d f<|j9|dz|dzd |jF!|j;td"|d d d n|r8t!|d5}|j#d(jO| dd d d |s| d)k(rd)}t j||d}t5|||| |d}td*|t%j"|||+|rtd,tQ||d#tStj|z d-\}}td.|d/d0|d1d2|S#1swYxYw#1swY7xYw#1swYxYw#1swYxYw#1swYxYw)3a Remove ramp from each 2D matrix of input file Parameters: fname - str, data file to be deramped ramp_type - str, name of ramp to be estimated. mask_file - str, file of mask of pixels used for ramp estimation out_file - str, output file name datasetName - str, output dataset name, for ifgramStack file type only save_ramp_coeff - bool, save the estimated ramp coefficients to text file extra_meta - dict, extra metadata to add to the output file Returns: out_file - str, output file name rhr r rz ramp from file: _ramprrzread mask file: r?zuse mask of the whole areaN rampCoeff_r\rz# input file: r~z# output file: z # ramp type: rT)ref_filerz,estimating phase ramp one date at a time ...)maxValuearri)r:rl coeff_filerF)rrblockr/)rzfinished writing to file: rrzaccess HDF5 dataset /)shaper@chunks compressionzcreate HDF5 dataset /z{} rz writing >>> )r2rJz*add/update the following metadata to file:<z time used: z02.0fz mins z02.1fz secs.)*timerBrCr]rDr_rrrrAr`rFrrJonesbool_rrnr layout_hdf5r get_date_listrr progressBarrrwrite_hdf5_blockrrrrrorcreate_dataset numIfgramrOrN date12Listformatrdivmod)rtr:r;r2rsave_ramp_coeff extra_meta start_timerbrcrrpftyper"r#rjrMrrXnum_dateprog_barrvrrds dsNameOutdsOutmss r*rHrHsJ 77??5 !D''""277#3#3E#:;KE4  ! !% (C  E X F G E GI;/w 78#xdugU4&$$h7x =A*c*5aGG ! .d345==IaLA!DD#%   D  & &(!A#q&!U3   OOAaC1Q3%q (;O <7 =8  *8*56 - %  5!'K YYuc "' 8a;B&-u-IAFFH$) -i[9:((==&%8** $ )& -i[9:((#--@H3==) Fj#.A!3s~~a'8#9":$ ?@A!Q'{' )   "&aAg!qse1S]]O,DE+ F, NN  .ug6 7O' 8' 8V j#& ;!K(89: ;u 2$K}}U rss4  OO33!.-)`7)t HLPTgT=D&R2jD+\"&55p#3P 5p!HMQ15ir,