B QaS@sddlZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z m Z m Z ddlmZmZmZddlTd.ddZd/d d Zd0d d Zd1ddZd2ddZd3ddZd4ddZd5ddZd6ddZd7dd Zedfd!d"Zd8d#d$Zd9d%d&Z d:d'd(Z!d;d*d+Z"dr?r@rArBrC) rDr.r-rEZrms_filerFrGZrms_listrHr#r#r$get_residual_rms\s.    rNmaskConnComp.h5cCsXt|}|d}|dkr,t|j|d}ntd|dSd|d<tj|||d|S)zDGenerate mask file for non-zero value of input multi-group hdf5 file FILE_TYPEr)rz5Only ifgramStack file is supported for now, input is Nmask)r)metadata)rrr nonzero_maskr2rwrite)Filer)ratrkrQr#r#r$rSs  rS coherenceFTc Csdd}t|} | d} |s:ddt| dt| df}| dkrF|ntjtj|d} |dkrjd nd } | |rxd nd 7} | | d } || rtd||\}}||fSd tj|}d |}d |}d |}yt | d}| }| |r,ydd|Dd}Wnd }YnXn|}ydd|Dd}Wnd }YnX||kr||krt | ||gdddkrt| d|| \}}||fSWn YnXd|krd}nd}| dkrt|}|j dd|j||||||d\}}|j}|j}| n| dkrsz;spatial_average..read_text_file..r cSsg|]}|qSr#r#)rYrZr#r#r$r[sr)r<r=r>r?r@)fnameZ txtContentmeanListdateListr#r#r$read_text_filesz'spatial_average..read_text_filerPrr r rNZ SpatialAvgZ AreaRatioZRevMskz.txtz;Input file is spatial average txt already, read it directlyz# Data file: {} z# Mask file: {} z# AOI box: {} z# Threshold: {} rcSsg|]}d|kr|qS)z # AOI box:r#)rYrZr#r#r$r[sz#spatial_average..cSsg|]}d|kr|qS)z # Mask file:r#)rYrZr#r#r$r[sF)r)r*r+skipz! already exists, read it directlyoffsetT) print_msg)rr/r useMedian reverseMask thresholdr)r/rrfrg)rzmask from file: rQ)rrr z-write average value in space into text file: wz'# DATE12 Mean Btemp/days Bperp/m Num z%s %.4f %8.0f %8.1f %d z# DATE12 Mean z%s %.4f )rrrr4r5r6basenameendswithr2r3open readlinescloser7rspatial_average pbaseIfgram tbaseIfgramrrr8r<nannanmeanrTlenrange) rUrr/rZsaveListZcheckAoirfrgr_rVrWprefixsuffixZtxtFiler]r^Z file_lineZ mask_lineZaoi_lineZ thres_lineZfllinesZ aoi_line_origZmask_line_origreobjpbasetbasedatarQZnumLinerZr#r#r$rns $                              *  rnc Csvtj||d}|d}|dkrBtd|t|d}||fS|dkrtj|d}|s|dkr|d krtd }qd |krd }qd |}nF|dkr||krtj| dd |d}d|}n d |}|rtj |rt|d} | |fS|dkr:t |j |d} d |kr0d|d<d|d<n||d<n|dkrXt | } d|d<|rntj| ||d| |fS)aCalculate temporal average of multi-temporal dataset, equivalent to stacking For ifgramStakc/unwrapPhase, return average phase velocity Parameters: File : string, file to be averaged in time datasetName : string, dataset to be read from input file, for multiple datasets file - ifgramStack - only e.g.: coherence, unwrapPhase updateMode : bool outFile : string, output filename None for auto output filename False for do not save as output file Returns: dataMean : 2D array outFile : string, output file name Examples: avgPhaseVel = ut.temporal_average('ifgramStack.h5', datasetName='unwrapPhase')[0] ut.temporal_average('ifgramStack.h5', datasetName='coherence', outFile='avgSpatialCoh.h5', updateMode=True) )rrP)rrzBWARNING: input file is not multi-temporal file: {}, return itself.rNr rrXzavgSpatialCoh.h5 unwrapPhasezavgPhaseVelocity.h5zavg{}.h5rzavgDisplacement{}.h5velocityzm/yearUNIT displacement)r)rR)rrr2r3rr4r5r6risplitr8rtemporal_averagerrrT) rUrZ updateModer0rVrWr{extZ processMarkZdataMeanr#r#r$r2sF          rc Cs|sgSt|tr|g}dd|D}g}xBtt|D]2}||}t|}|ttt|t|7}q8W|rdd|D}|dk rxlt|D]`}t |}|dkrd| kr| |q|dkrd| kr| |qd |} t | qW|S) aRGet all existed files matching the input list of file pattern Parameters: file_list - string or list of string, input file/directory pattern abspath - bool, return absolute path or not coord - string, return files with specific coordinate type: geo or radar if none, skip the checking and return all files Returns: file_list_out - list of string, existed file path/name, [] if not existed Example: file_list = get_file_list(['*velocity*.h5','timeseries*.h5']) file_list = get_file_list('timeseries*.h5') cSsg|]}|dkr|qS)Nr#)rYxr#r#r$r[sz!get_file_list..cSsg|]}tj|qSr#)r4r5rL)rYrZr#r#r$r[sN)geor)ZradarZrdrZrdcz"un-recognized input coord type: {}) isinstancer@rtrsglobsortedrCsetrrrremover3 ValueError) file_listrLcoordZ file_list_outrZZfile0Z file_list0r\rVrFr#r#r$ get_file_listts0           rc s|sHdddddddg}dd d g}g}x"|D]|fd d |D7}q*Wg}y t|}Wn&|r|td tdt|dSd}xP|D]H}t|}x8dD]0} ytj|| ddd} |}PWqYqXqWqW|s|rtddS|rtj|}|S)aFind lookup table file with/without input file pattern Parameters: filePattern - list of str abspath - bool, return absolute path or not print_msg - bool, printout message or not Returns: outFile - str, path of the lookup file zgeometryRadar.h5zgeometryGeo_tight.h5zgeometryGeo.h5zgeomap*lks_tight.transzgeomap*lks.transzsim*_tight.UTM_TO_RDCzsim*.UTM_TO_RDCinputsr`z ../inputscsg|]}tj|qSr#)r4r5rJ)rYr\)rKr#r$r[sz#get_lookup_file..z-ERROR: No geometry / lookup table file found!zIt should be like:N)r rangeCoordF)rrdrz.No lookup table info range/lat found in files.)rr2rrrr4r5rL) Z filePatternrLrdZfileListZdirListZ existFilesr0r\rVdsNamedsetr#)rKr$get_lookup_filesF       rrcst|tr|g}x"|D]}|tkrtd|qWs@tfdddD}t||d}t|dkrz|rvt ddSx0t |D]$t fd d |Dr| qWt|dkr|rt d |dS|d}|rtj |}|S) z4Find geometry file containing input specific datasetz&unrecognized geometry dataset name: {}csg|]}tj|qSr#)r4r5rJ)rYrZ)work_dirr#r$r[sz%get_geometry_file..)z *geometry*.h5z*/*geometry*.h5z../*/geometry*.h5)rrzNo geometry file found.Nc3s|]}|tkVqdS)N)rget_dataset_list)rYr)r\r#r$ sz$get_geometry_file..z&No geometry file with dataset {} found)rr@rrr3r4getcwdrrsr2rCanyrr5rL)Z dset_listrrrLrdr fname_listrr#)r\rr$get_geometry_files0      r=c Csld}t|}x0|D]$\}}||kr|||krd}qW|sTtd||S|d}t|d}xt|dD]} dd| |d D} | d sFt | d krF| d }t | d d d dd }| dd}||krF|||krFt |d|| d } | |||} | | | d } td|||||| qrW|t|||S)zEUpdate option value in template_file with value from input extra_dictFTz)No new option value found, skip updating z.tmprhracSsg|] }|qSr#)strip)rYrZr#r#r$r[sz(update_template_file..r )%#r r`rr z\*z[\s]*z {}: {} --> {})r read_templateitemsrr2rkrr startswithrsr@replacerefindallr3rTrmshutilmove) template_fileZ extra_dict delimiterupdateZ orig_dictkeyvalueZtmp_fileZf_tmplinecZ old_value_strZ new_value_strr#r#r$update_template_files2      rcCst|}t||}|s$td|St|d}xt|D]p\}}|dksV|dkry"|j ||rvtd |WqYqXq>t ||j|<|r>td |t |q>W| |S)a#Add/update input attribute into File Parameters: File - string, path/name of file atr_new - dict, attributes to be added/updated if value is None, delete the item from input File attributes Returns: File - string, path/name of updated file zeAll updated (removed) attributes already exists (do not exists) and have the same value, skip update.zr+NoneNz remove {}z{} = {}) rrupdate_attribute_or_notr2h5pyrUiterrattrspopr3r@rm)rUatr_newrdrVrfrrr#r#r$ add_attributes&    rc Cs@|s|ddfSg}g}x2|D]*}t|}||d||dqW|sVt|}|sbt|}t|}||t|ks||t|kr6tdtdtdtd||ftdxXtt|D]H}|||ks|||krtd ||||||f| ||qWtd t t|td |||fS) zXCheck file size in the list of files, and drop those not in the same size with majority.Nr r z-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%z2WARNING: Some files may have the wrong dimensions!z$All files should have the same size.z9The width and length of the majority of files are: %s, %szNBut the following files have different dimensions and thus will not be loaded:z%s width: %s length: %sz Number of files left: z,%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%) rrappend most_commonrCcountrsr2rtrr@) rZ mode_widthZ mode_lengthZ width_listZ length_listr\rVZfname_list_outrZr#r#r$check_file_size=s:    rcCs4yt||dd}t|Wnd}YnX|S)aCheck if any file in the file list 1) exists and 2) readable Parameters: file_list : list of string, file name with/without wildcards abspath : bool, return absolute file name/path or not Returns: file_path : string, found file name/path; None if not. )rLrN)rrr)rrLfiler#r#r$ is_file_exisths  rcCs|sdSt|tr|g}tdd|Ds.dS|r|yt|d}|d}Wn.|rjtd|dt|ddS|rt |}|rt dd|D}t d d|D}||krdS|rtd ||d S) aCheck whether to update out_file or not. return run if any of the following meets: 1. out_file is empty, e.g. None, [] 2. out_file is not existed 3. out_file is not readable by readfile.read_attribute() when check_readable=True 4. out_file is older than in_file, if in_file is not None Otherwise, return skip. If in_file=None and out_file exists and readable, return skip Parameters: out_file : string or list of string, output file(s) in_file : string or list of string, input file(s) check_readable : bool, check if the 1st output file has attribute 'WIDTH' print_msg : bool, print message Returns: run/skip : str, whether to update output file or not Example: if ut.run_or_skip(out_file='timeseries_ERA5_demErr.h5', in_file='timeseries_ERA5.h5'): if ut.run_or_skip(out_file='exclude_date.txt', in_file=['timeseries_ERA5_demErrInvResid.h5', 'maskTempCoh.h5', 'smallbaselineApp.cfg'], check_readable=False): r,css|]}tj|VqdS)N)r4r5r8)rYrZr#r#r$rszrun_or_skip..rr z'{} exists, but can not read, remove it.cSsg|]}tj|qSr#)r4r5getmtime)rYrZr#r#r$r[szrun_or_skip..cSsg|]}tj|qSr#)r4r5r)rYrZr#r#r$r[sz({} exists and is newer than {} --> skip.rb) rr@allrrr2r3r4rrmaxmin)r)r*r+rdrVwidthZt_inZt_outr#r#r$r7vs2  r7"defaults/smallbaselineApp_auto.cfgc Cstjtjtj|}t|}d}||d }|dkrF||}|dkrVd|d<x4| D](\}}|dkr`|| kr`||||<q`Wdddddd }x4| D](\}}| }|| kr||||<qW|S) z7Replace auto value based on the input auto config file.zmintpy.compute.clusterautolocalZ40zmintpy.compute.numWorkerTFN)yestruer(falsenone) r4r5rJrKmintpy__file__rrgetlowerrr) Z templateDictZ auto_fileZtemplateAutoFileZtemplateAutoDictZ cluster_keyZclusterrrZ specialValuesr#r#r$check_template_auto_values*  rc CsZt}t|}|d}t|d} t|d} td|||sbtj|\} } d| | }|dkrn|}tj |rt |d} td|nt j | | ft j d } td d }|r6tjtj|d} tjtj| d | }t|d 6}|d||d||d|Wd QRX|dkrDtj||ddtdt|}t|}tj|d}xt|D]}|rt|d}|d||Wd QRXtj |||dd}t|| |||dd}tj||d||dd| d| gdd|j|dd|d|dqW|td|n|dkrt|}|jdd |sld!}t !|d<}||}d"|}||"kr||}td#|n,|j#||j$| | ft j%dd d$}td%|tj|j$d}xt|j$D]}|r4t|d }|dt&|j'|Wd QRX||d d d d f}t|| |||dd}|||d d d d f<|j|dd|d|j$dqW|td|Wd QRXnr|rt|d}|d|dWd QRXt |d}t|| |||dd}td&|tj|||d't(t|d(\}}td)|||S)*a Remove ramp from each 2D matrix of input file Parameters: fname : str, data file to be derampped ramp_type : str, name of ramp to be estimated. mask_file : str, file of mask of pixels used for ramp estimation out_file : str, output file name datasetName : str, output dataset name, for ifgramStack file type only save_ramp_coeff : bool, save the estimated ramp coefficients to text file Returns: out_file : str, output file name rPr r zremove {} ramp from file: {}z {}_ramp{}rrzread mask file: )r1zuse mask of the whole areaNzrampCoeff_{}.txtrhz# input file: {} z# output file: {} z# ramp type: {} rT)ref_filerdz,estimating phase ramp one date at a time ...)maxValueaz{} )r)r-rR coeff_filer F)rblockrdz{}/{})rvzfinished writing to file: {})rdr|z{}_rampzaccess HDF5 dataset /{})shaper1chunks compressionzcreate HDF5 dataset /{}zwriting >>> {})r)r<z'time used: {:02.0f} mins {:02.1f} secs.))timerrrr2r3r4r5r6r8rr<onesbool_rirJrKrkrTr layout_hdf5r get_date_listrsr progressBarrtrwrite_hdf5_blockrrmrrrUrcreate_dataset numIfgramrAr@ date12Listdivmod)r\r-r.r)rZsave_ramp_coeff start_timerVrWlengthrfbasefextrQrrrHnum_dateprog_barrZr{rxdsZ dsNameOutZdsOutmsr#r#r$r:s            $     $ &  r:)N)r&r')r&r')rON)rXNNFTFN)rXFN)FN)NFT)NrTT)r)NN)T)NTT)r)NNNF)$r4rrrrrnumpyr<rmintpy.objectsrrrr mintpy.utilsrrrmintpy.utils.utils0r%rIrNrSrnrrrrrdictrrrr7rr:r#r#r#r$ s8  , 3   B * 5 " %# +  9 #