o
    8Pe                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dlZd dl	Z	d dl
Zd dlmZ d dlmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZ i ddd	d
dddddddddddddddddddddddd d!d"d#d$ZG d%d& d&ZG d'd( d(eZ d)d* Z!ded,d-Z"	.	dfd/d0Z#dgd1d2Z$dhd3d4Z%did6d7Z&d8d9 Z'djd;d<Z(d=d> Z)d?d@ Z*dkdAdBZ+dldCdDZ,dEdF Z-dGdH Z.dIdJ Z/dKdL Z0dMdN Z1dOdP Z2dQdR Z3dSdT Z4ddVdWZ5dmdYdZZ6d[d\ Z7d]d^ Z8d_d` Z9dndadbZ:dodcddZ;dS )p    N)
coordinate)gdal)MiaplPyParser)readfileptimeutils)
DSET_UNIT_DICTgeometryGEOMETRY_DSET_NAMESgiantIfgramStackgiantTimeseriesIFGRAM_DSET_NAMESifgramStackTIMESERIES_DSET_NAMES
timeseriesHDFEOSslcmiaplpy.load.slcFileunwrapPhasezmiaplpy.load.unwFile	coherencezmiaplpy.load.corFileconnectComponentzmiaplpy.load.connCompFile	wrapPhasezmiaplpy.load.intFileZionozmiaplpy.load.ionoFileheightmiaplpy.load.demFilelatitudemiaplpy.load.lookupYFile	longitudemiaplpy.load.lookupXFileazimuthCoord
rangeCoordincidenceAnglezmiaplpy.load.incAngleFileazimuthAnglezmiaplpy.load.azAngleFile
shadowMaskzmiaplpy.load.shadowMaskFile	waterMaskzmiaplpy.load.waterMaskFilebperpzmiaplpy.load.bperpFilec                   @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )
OutControlc                 C   s   t j|| _|| _d S N)ospathabspathrun_filework_dir)self
batch_filer+    r.   O/home/exouser/operations/rsmas_insar/tools/MiaplPy/src/miaplpy/objects/utils.py__init__4   s   
zOutControl.__init__c                 C   s$   |    |   |   |   d S r&   )&remove_zero_size_or_length_error_filesraise_exception_if_job_exitedconcatenate_error_filesmove_out_job_files_to_stdoutr,   r.   r.   r/   clean8   s   zOutControl.cleanc                 C   s`   t  | jd }t  | jd }t  | jd }|| | }t|dks.|D ]}t| q&d S )N*.e*.job*.or   )globr*   lenr'   remove)r,   error_files	job_filesout_fileZ
list_filesitemr.   r.   r/    remove_last_job_running_products>   s   z+OutControl.remove_last_job_running_productsc                 C   sX   t  | jd }|  |D ]}tj|dkrt| qt|dkr)t| qdS )ERemoves files with zero size or zero length (*.e files in run_files).r7   r   N)r:   r*   sortr'   r(   getsizer<   file_len)r,   r=   r@   r.   r.   r/   r1   H   s   
z1OutControl.remove_zero_size_or_length_error_filesc              	   C   s|   t  | jd }d}|  |D ]+}t|}| }|D ]}||v r+td||qW d   n1 s6w   Y  qdS )rB   r9   zExited with exit codez ERROR: {0} exited; contains: {1}N)r:   r*   rC   open	readlines	Exceptionformat)r,   filesZsearch_stringfilefrlinesliner.   r.   r/   r2   T   s   
z(OutControl.raise_exception_if_job_exitedc              
   C   s:  t j| jd | jdd  d }t j|rt | t j| jd | jdd  d }t		| jd }t
|dkst|d@}|D ]5}|d |d	| d
  |d t|}||  W d   n1 srw   Y  t | qGW d   n1 sw   Y  tt j|t j| j dS )z
        Concatenate error files to one file (*.e files in run_files).
        :param directory: str
        :param out_name: str
        :return: None
        z/out_/z.er7   r   wz#########################
z#### z 
N)r'   r(   r)   r+   r*   splitisfiler<   dirnamer:   r;   rF   writereadshutilmove)r,   r?   out_namer=   outfilefnameinfiler.   r.   r/   r3   e   s&   &
&


	z"OutControl.concatenate_error_filesc                 C   s   t  | jd }t  | jd }tj|d }|d tj| j }tj|s/t| n
t	| t| |D ]}t
|| q;|D ]}t
|| qFdS )z/move the error file into stdout_files directoryr8   r9   r   z/stdout_N)r:   r*   r'   r(   rT   basenameisdirmkdirrW   rmtreerX   )r,   r>   stdout_filesZdir_nameZ
out_folderr@   r.   r.   r/   r4      s   

z'OutControl.move_out_job_files_to_stdoutN)
__name__
__module____qualname__r0   r6   rA   r1   r2   r3   r4   r.   r.   r.   r/   r%   3   s    
r%   c                       s0   e Zd Zd	 fdd	Zdd Zd
ddZ  ZS )	coord_revNc                    s   t  || d S r&   )superr0   )r,   metadatalookup_file	__class__r.   r/   r0      s   zcoord_rev.__init__c                 C   s   z
t | jd | _W n   d| _Y d| j v r>d| _t | jd | _t | jd | _t | jd | _t | jd | _d S d| _| j	rPt
| j	d	 d
d| _d S d S )NEARTH_RADIUSg    MXAY_FIRSTTX_FIRSTY_STEPX_STEPFr   .xmlmetafile_ext)floatsrc_metadataearth_radiuskeysgeocodedlat0lon0lat_steplon_steprh   read_attributelut_metadatar5   r.   r.   r/   rF      s   zcoord_rev.openTc                 C   s   d| j  v r$tj| jd d|dd | _tj| jd d|dd | _n td| jd  t| jd | _td| jd  t| jd | _| j| jfS )	Nrl   r   r   )datasetName	print_msg   r   zLoading  .... zLoading .... )	r}   rv   r   rV   rh   lut_ylut_xprint
read_image)r,   r   r.   r.   r/   read_lookup_table   s$   zcoord_rev.read_lookup_tabler&   )T)rb   rc   rd   r0   rF   r   __classcell__r.   r.   ri   r/   re      s    re   c                 C   s:   t | d}t| W  d   S 1 sw   Y  dS )z(Calculate the number of lines in a file.rN)rF   r;   rG   )r[   rK   r.   r.   r/   rE      s   
$rE   Tc                    s  t jt j| \}}| }|du r| }n| | }t j|s2d| }|dt   7 }t||dv rt	
| dfdd D fdd D  g d	}t fd
ddD red}	nt fdddD rsd}	n|t fdddD rd}	nmt fdddD rd}	n^t fdddD rd}	nPdv rd}	nId v rd}	nBt fdddD rd}	n4tfd d|D rttt|@ d! }	nt d!krވ d! }	ntd!krd! }	ntd"|  |	dkrt|  }
nD|	dkrt|  }
n8tjd!krd#j v rtj}
n"d$d% }g a| tr9d&d tD }tt| }
ntd'| |
 D ]\}}z
|d(|
|< W qB   ||
|< Y qBt|	|
d)< d}|rr| v rr| }nd*d+ }g a| trtd! }|durt|j|
d,<   d-|
 v r|
d- |
d.< d.|
 vrd/|
d.< n| d0 | d1 | d2 t j| d! d3 | d4 | d5 g}d6d |D }t|d!krt d7| i }
| !d8rtd9d |D rd:|
d.< nKtd;d |D rd<|
d.< d=d |D }t|d!kr|
"t#$|d!  n$td>d |D r*d?|
d.< ntd@d |D r?d.|
 vr?dA|
d.< d.|
 vrJd/|
d.< |d! }|dBv r`t j|\}}|dBv sS|se|}|!d0r|
"t#%| d)|
 vr~||
d)< n|!d1r|
"t#$| d)|
 vr||
d)< ny|!d2r|
"t#&| ||
d)< nf|!d3r |
"t#'| |
d. d:krt j|  }|(dCrdD|
d)< n>|(dErdF|
d)< n3|(dGrdH|
d)< n(dI|v rdI|
d)< n|
dJ |
d)< n||
d)< n|!d4r|
"t#)| ||
d)< dKdLdMdNdO}|
*d,dP }|dPkr3|| v r3|| |
d,< |
d) +dQdR}	|	dkrV|rQ|t, v rQt,| |
dS< n/dT|
dS< n*|rg|t, v rgt,| |
dS< ndS|
 vr|	t, v r|t,|	 |
dS< ndU|
dS< |
d) +dQdR}	|	dkrdV|
dS< n*dS|
 vr|r|t, v rt,| |
dS< n|	t, v rt,|	 |
dS< ndU|
dS< t j-| |
dW< |rt#.|
}
|
S )Xaa  Read attributes of input file into a dictionary
        Parameters: fname : str, path/name of data file
                    datasetName : str, name of dataset of interest, for file with multiple datasets
                        e.g. slc         in slcStack.h5
                             date        in slcStack.h5
                             height      in geometryRadar.h5
                             latitude    in geometryRadar.h5
                             ...
                    standardize : bool, grab standardized metadata key name
        Returns:    atr : dict, attributes dictionary
        Nzinput file not existed: {}
zcurrent directory: z.h5z.he5r   c                        g | ]}t  | tjr|qS r.   )
isinstanceh5pyGroup.0ifr.   r/   
<listcomp>        z"read_attribute.<locals>.<listcomp>c                    s.   g | ]}t  | tjr | jd kr|qS )   )r   r   Datasetndimr   r   r.   r/   r      s   . )interferogramsr   wrappedc                 3       | ]}| v V  qd S r&   r.   r   d1_listr.   r/   	<genexpr>       z!read_attribute.<locals>.<genexpr>)r   azimuthOffsetr   c                 3   r   r&   r.   r   r   r.   r/   r      r   )r   r   r   r	   c                 3       | ]	}|  v V  qd S r&   r.   r   r   g1_listr.   r/   r          )r   displacementr   c                 3   r   r&   r.   r   r   r.   r/   r      r   r   r   c                 3   r   r&   r.   r   r   r.   r/   r      r   )velocityr   r   reconsr   c                 3   r   r&   r.   r   r   r.   r/   r      r   )igramfigramr   c                 3   r   r&   r.   r   )r   r.   r/   r      r   r   zunrecognized file type: WIDTHc                 S   s8   t |jdkrd|j v rtt|j d S d S d S )Nr   r   )r;   attrsrv   atr_listappenddictnameobjr.   r.   r/   get_hdf5_attrs  s   z&read_attribute.<locals>.get_hdf5_attrsc                 S      g | ]}t |qS r.   r;   r   r.   r.   r/   r         z!No attribute WIDTH found in file:utf8	FILE_TYPEc                 S   s,   t |tjr|jdkrt| d S d S d S )Nr   )r   r   r   r   ds_listr   r   r.   r.   r/   get_hdf5_dataset0  s   z(read_attribute.<locals>.get_hdf5_dataset	DATA_TYPEINSAR_PROCESSOR	PROCESSORmintpy.rscrp   .par.hdr.vrtz.aux.xmlc                 S   s   g | ]
}t j|r|qS r.   )r'   r(   rS   r   r.   r.   r/   r   O      z(No metadata file found for data file: {}z.imgc                 s       | ]}| d V  qdS )r   Nendswithr   r.   r.   r/   r   U      snapc                 s   r   ))rp   r   r   Nr   r   r.   r.   r/   r   X  r   iscec                 S   s   g | ]	}| d r|qS )rp   r   r   r.   r.   r/   r   Z      c                 s   r   )r   Nr   r   r.   r.   r/   r   ^  r   gammac                 s   r   )r   Nr   r   r.   r.   r/   r   a  r   roipac)z.geoz.rdrz.fullunw.unw)cohcor.cor	phase_ifg.intdemz	file typeint8float32float64	complex64byters   doublecfloatnone. UNITradian1r   	FILE_PATH)/r'   r(   splitextr]   lowerrS   rI   getcwdrH   r   Filerv   anylistsetr;   
ValueErrorr   get_metadatar   r   r   r   
visititemsnpargmaxitemsdecodestrr   dtypecloseFileNotFoundErrorr   updater   read_isce_xmlread_roipac_rscread_gamma_parread_envi_hdr
startswithread_gdal_vrtgetreplacer   r)   standardize_metadata)r[   r~   Zstandardizerr   fbasefextZ	test_filemsgpy2_mintpy_stack_fileskatrr   num_listkeyvaluedsr   	metafiles	xml_filesmetafiledataTypeDict	data_typer.   )r   r   r   r/   r|      sL  




 




 















r|   ../defaults/miaplpyApp_auto.cfgc                 C   s  t jt jt|}t|}|dur!| D ]\}}|| |< q|  D ]\}}|dkr9|| v r9|| | |< q%ddg}|durCd}	| d dkrm| d dkrm|d	 dks_|d	 | d< d
}	|d dksm|d | d< d
}	|D ]}| d|  dkr|d|  dvr|d|  | d| < d
}	qo| d |d ks| d dkr|d | d< d
}	dd |D ddg }|	d
krC|d }
t	|
d}t	|dD ]v}dd |
 ddD }|ds3t|dkr3|d }||v r3| | r3| | }|dkrd}n|dkrd}d| }t|d d d!d"d 
 }|d#d$}td%| |d }|||d}td&||| |  || q|  t|
| dddddd'}|  D ]\}}|| v r`|| | |< qO| S )(z7Replace auto value based on the input auto config file.Nautozload.autoPathzload.compressionskipmiaplpy.subset.lalonomiaplpy.subset.yxmintpy.subset.lalorunmintpy.subset.yxmiaplpy.zmintpy.)r  Fmiaplpy.load.processorzmintpy.load.processorr   c                 S   s   g | ]}d | qS )r  r.   r   r  r.   r.   r/   r     r   z-check_template_auto_value.<locals>.<listcomp>.tmprQ   r   c                 S      g | ]}|  qS r.   stripr   r.   r.   r/   r     r   =r   %#r   TyesFz= 
r   r   *z\*z=[\s]*    {}: {} --> {})r!  Truer  Falser   )r'   r(   joinrT   __file__r   read_templater   rv   rF   r  rR   r   r;   r   r   refindallr   rI   rU   r   rW   rX   )templateDictZmintpyTemplateDict	auto_filetemplateFiletemplateAutoFiletemplateAutoDictr  r  Zcommon_keysstatustmp_filef_tmprN   cZs_valuenew_value_strold_value_strZspecialValuesr.   r.   r/   check_template_auto_value  s~   




 r7  c                    s   d}t |tr|d dd }nt |tr|dd } fdddD }dd |D d }t ||d}t|d	 t|d
 }}	|sKdd|	|f}tjtj	 d 
 }
|
dv rht ||d}||fS t ||d\}}||fS )a  Read one dataset and its attributes from input file.
    Parameters: fname : str, path of file to read
                datasetName : str or list of str, slice names
                box : 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate
    Returns:    data : 2/3-D matrix in numpy.array format, return None if failed
                atr : dictionary, attributes of data, return None if failed
    Examples:
        from mintpy.utils import readfile
        data, atr = readfile.read('velocity.h5')
        data, atr = readfile.read('timeseries.h5')
        data, atr = readfile.read('timeseries.h5', datasetName='timeseries-20161020')
        data, atr = readfile.read('ifgramStack.h5', datasetName='unwrapPhase')
        data, atr = readfile.read('ifgramStack.h5', datasetName='unwrapPhase-20161020_20161026')
        data, atr = readfile.read('ifgramStack.h5', datasetName='coherence', box=(100,1100, 500, 2500))
        data, atr = readfile.read('geometryRadar.h5', datasetName='height')
        data, atr = readfile.read('geometryRadar.h5', datasetName='bperp')
        data, atr = readfile.read('100120-110214.unw', box=(100,1100, 500, 2500))
    Nr   -c                    s   g | ]} | qS r.   r.   r   r[   r.   r/   r   /  r   zread.<locals>.<listcomp>)r   rp   c                 S   s*   g | ]}t j|rd |d d  qS )r   rP   )r'   r(   existsrR   r   r.   r.   r/   r   0  s   * )r~   rr   LENGTHr   r   r   )r~   box)r   r   rR   r   r|   intr'   r(   r   r]   r   read_hdf5_fileread_binary_file)r[   r<  r~   r   
dsname4atrrJ   Zfext0r  lengthwidthr   datar.   r9  r/   rV     s"   

rV   c                    sP  t | }g }dd |D D ]}||vr|| qdd |D fdd|D  s1|d g nt tr9 g tdd  D rKfdd D   d d	d fd
d D }t| dfdd d fD } fdd|D }t|dkr|d  }	nt|dkr|d  }	nt	d
 | |	jdkr|	|d |d |d |d f }
na|	jdkrtj|	jd tjd}|r|dgkrd|dd< ndd fdd|D D }|D ]	}d|||< q|	||d |d |d |d f }
t|
}
W d   |
S W d   |
S W d   |
S 1 s!w   Y  |
S )a7  
    Parameters: fname : str, name of HDF5 file to read
                datasetName : str or list of str, dataset name in root level with/without date info
                    'timeseries'
                    'timeseries-20150215'
                    'unwrapPhase'
                    'unwrapPhase-20150215_20150227'
                    'HDFEOS/GRIDS/timeseries/observation/displacement'
                    'recons'
                    'recons-20150215'
                    ['recons-20150215', 'recons-20150227', ...]
                    '20150215'
                    'cmask'
                    'igram-20150215_20150227'
                    ...
                box : 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate
    Returns:    data : 2D/3D array
                atr : dict, metadata
    c                 S      g | ]	}| d d qS )r8  r   rR   r   r.   r.   r/   r   [  r   z"read_hdf5_file.<locals>.<listcomp>c                 S   s   g | ]}d |vr|qS )r8  r.   r   r.   r.   r/   r   ^      c                    s   g | ]}| vr|qS r.   r.   r   )
ds_2d_listr.   r/   r   _  rF  r   c                 s   s    | ]}|  V  qd S r&   )isdigitr   r.   r.   r/   r   f  r   z!read_hdf5_file.<locals>.<genexpr>c                    s   g | ]
}d   d |qS )z{}-{}r   rI   r   )
ds_3d_listr.   r/   r   g  r   r8  c                    s    g | ]}|  d  dd qS )r   r8  )r   r   dsFamilyr.   r/   r   j  r   r   c                       g | ]
}|   v r|qS r.   rv   r   r   r.   r/   r   o  r   c                    s"   g | ]}d   d |v r|qS )z/{}r   rI  r   )r~   r.   r/   r   p     " z%input dataset {} not found in file {}r   r      )r   r   TNc                 S   rD  )r8  r   rE  r   r.   r.   r/   r     r   c                    s   g | ]	}|  r|qS r.   )r   )r   jrK  r.   r/   r     r   )get_slice_listr   r   r   allrR   r   r   r;   r   rI   r   r   zerosshapebool_indexsqueeze)r[   r~   r<  
slice_listr   r   inputDateListdsNames
dsNamesOldr  rC  
slice_flag	date_listdr.   )r~   rL  rG  rJ  r   r/   r>  D  s^   


&&


r>  Fc                 C   s  t jt j| \}}| }t| dd}|d }t|d }t|d }	|s.dd|	|f}|dd }
|d	d
 }t|dd}|dd }d}d}|dv rddddd}|
|	 v rg||
 }
|d  
dd}|dv rtd|}|r|dv rd}ni|dv rd}nb|dv r|r|drtd|}nQ|d v rtd|}|rd!| vrd}n=|r| d"krd}n2| d#krd$}n)|d%rd}n!|d&v rd}n| d'krd'}n| d(rd(}n|d)rd*}t||}n|d+v r>d}d
}d}
d}|d,v r	d}d}n|d-v rd}
n|d.v rd}
d}n|d/v r#d0}
n|d1v r+d2}
n|d3v r=d}|r=|d4r=d}nm|d5krd}|d	d6}d}
|d7v rSnX|d-v r[d}
nP|d8v rqd}
|rn|d4rnd(}n=d'}n:|d9kr{d:}
d}n0|d;v rd
}n(|d<kr|dd= }|d	d6}d>|	 v r|d> d?krd
}ntd@ d|vr|
|d< |r|S t| ||dA}||fS )Ba#  Read data from binary file, such as .unw, .cor, etc.
    Parameters: fname : str, path/name of binary file
                datasetName : str, dataset name for file with multiple bands of data
                    e.g.: incidenceAngle, azimuthAngle, rangeCoord, azimuthCoord, ...
                box  : 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate
    Returns:    data : 2D array in size of (length, width) in BYTE / int16 / float32 / complex64 / float64 etc.
                atr  : dict, metadata of binary file
    rp   rq   r   r;  r   r   r   r   
BYTE_ORDERzlittle-endiannumber_bandsr   schemeBILr   phase)r   r   r   r   r   r   r   r   )r   r   r   )band1	intensity	magnituder   rg  )los)band2azhead)incLocallocalri  band3rP  )magamp)rd  anglerealimag)cpxcomplexru  )r   )r   r   z.hgtz.msk)r   ).amp).demz.wgs84int16)z.flgz.bytrV  ).trans)rj  azimuthr   z
big-endian)r   r   z.hgt_simrw  rv  z.ramp).utm_to_rdcz.slc	complex32)z.mlir   BSQz
byte order0zUnknown InSAR processor.)r<  band)r'   r(   r   r]   r   r|   r=  r   upperrv   r   minr   r   r   )r[   r~   r<  Zattributes_onlyr   r   r  	processorrA  rB  r  
byte_ordernum_bandZband_interleaver  cpx_bandr  r  rC  r.   r.   r/   r?    s   


















r?  c                    s`  t jt j| \}}| }t| }|d }|dv rt| d  fdd  D }W d   n1 s8w   Y  |dkrT||v rTt	| }|j
dd	 |jatS |d
krgt| }|j
dd	 |jatS |dv r~||vr~t| }|j
dd	 |jatS |dv rt| }|j
dd	 |jatS |dv rt| }|j
dd	 |jatS |dv rt| }|j
dd	 |jatS |dv rt| }|j
dd	 |jatS t|d t|d fdd}g at| d  | W d   tS 1 sw   Y  tS | dv rddgatS |drddgatS |dddkr+d|vr+ddgatS d gatS )!z2Get list of 2D slice existed in file (for display)r   r   r   c                    r   r.   )r   r   r   r   r   r.   r/   r   @  r   z"get_slice_list.<locals>.<listcomp>Nr   Fr   r   )r	   )r   )r   )r   )r   r;  r   c                    sZ   t |tjr)|jdd   fkr+|jdkrt|  d S td	|j|  d S d S d S )Nr   z#file has un-defined {}D dataset: {})
r   r   r   rU  r   rY  r   warningswarnrI   r   )rA  rB  r.   r/   get_hdf5_2d_dataseth  s
   "
z+get_slice_list.<locals>.get_hdf5_2d_dataset)ry  r{  r   r   rh  r    r!   ra  r   2r   re  ri  r   )r'   r(   r   r]   r   r|   r   r   rv   r   rF   	sliceListrY  slcStackr	   r   r   r   r   r=  r   r   r   )r[   r   r   r  r  r   r   r  r.   )r   rA  rB  r/   rR  5  s~   :50+&!
rR  r   c                 C   s<   ddl m} || }dd ||D }|dkr|d }|S )zReturn the k most common item in the list L.
    Examples:
        5, 8 = most_common([4,5,5,5,5,8,8,8,9], k=2)
        'duck' = most_common(['goose','duck','duck','dog'])
        'goose' = most_common(['goose','duck','duck','goose'])
    r   )Counterc                 S      g | ]}|d  qS )r   r.   r   r.   r.   r/   r     r   zmost_common.<locals>.<listcomp>r   )collectionsr  most_common)Lr  r  cntitem_mmr.   r.   r/   r    s   r  c                 C   s   | d }| d }t d t d| t d| | d }| dds)| d	 }n|}| d
 | d f}| dds<d}n|}t d|d |d  ||||||fS )N
updateModecompression2--------------------------------------------------zupdateMode : {}zcompression: {}r<  rw   Fbox4geo_lutxstepystep)r   r   zx/ystep: {}/{}r   r   )r   rI   r   )iDictr  compr<  ZboxGeoZxyStepZ	xyStepGeor.   r.   r/   print_write_setting  s   
r  c                 C   sV   t tj| dd}tjtj d}|d | }t| ||d  |	  d S )Nlogza+z%Y%m%d:%H%M%Sz * r"  )
rF   r'   r(   r'  datetimestrftimenowr   rU   r   )Zlogdirr   r   dateStrstringr.   r.   r/   log_message  s   r  c                 C   sf   t | d t j}|dur)||}| |d |d |d |d f }~|S || }~|S )z Reads images from isce. r   Nr   rP  r   r   )r   OpenGA_ReadOnlyGetRasterBandReadAsArray)Z
image_filer<  r  r  Zimdsimager.   r.   r/   r     s   
(r   c                 C   s<   ddl m} | }ddl}|j|}|j| |}||fS )zL create a custom colormap based on visible portion of electromagnetive wave.r   )rgbN)Zmiaplpy.spectrumRGBr  
matplotlibcolorsZListedColormapZ	Normalize)vminvmaxr  mplcmapnormr.   r.   r/   custom_cmap  s   r  c                    s   ddl }ddl}td}d}t }tj| d g d} fdd|D }t| d	}d}	|D ]}
|	d
 }	|d |
 }q1d| d | d | d | }d| d | d }t| |j|dd	 }|durn|
d dS )z email mintpy results r   NZNOTIFICATIONEMAILzemail mintpy resultspic)zavgPhaseVelocity.pngzavgSpatialCoh.pngzgeo_maskTempCoh.pngzgeo_temporalCoherence.pngzgeo_velocity.pngzmaskConnComp.pngzNetwork.pdfzBperpHistory.pdfzCoherenceMatrix.pdfzrms_timeseriesResidual_ramp.pdfzgeo_velocity.kmzc                    s   g | ]	}t j |qS r.   )r'   r(   r'  r   pic_dirr.   r/   r     r   z!email_miaplpy.<locals>.<listcomp>r   r   z -a zecho "z" | mail -s  zssh pegasus.ccs.miami.edu "cd z; "T)shellzError in email_miaplpy)
subprocesssysr'   getenvr   r(   r'  r   Popenwaitexit)r+   r  r  Zemail_addressZtextStrcwdflist	file_listZattachmentStrr   fileListZmailCmdcommandr1  r.   r  r/   email_miaplpy  s*   
 
r  c                    s  ddl m} 	 tjtjtd}tj| d}tj|s.td	| t
||  |S ||j ||j}t fdd| D rtd t
||  ||j}| D ]\}}|  v rk | |krkd	}qXd
}qX|sxtd|  |S |d }	t|	d}
t|dD ]S}dd | ddD }|dst|dkr|d }t|d dddd  }|  v rԈ | |kr|| | d}td	|| |  |
| q|
  t
|	| |S )Nr   )Templatez../defaults/miaplpyApp.cfgmiaplpyApp.cfgz/copy default template file {} to work directoryc                    s   g | ]}|   vqS r.   rN  r  cdictr.   r/   r     rF  z/get_latest_template_miaplpy.<locals>.<listcomp>zAobsolete default template detected, update to the latest version.TFz)No new option value found, skip updating r  rQ   r   c                 S   r  r.   r  r   r.   r.   r/   r     r   r  r   r  r"  r   r   r$  )Zmiaplpy.objects.read_templater  r'   r(   r'  rT   r(  rS   r   rI   rW   copy2optionsr   rv   r   rF   r  rR   r   r;   r   r   rU   r   rX   )r+   r  lfilecfileldict	orig_dictr  r  r   r2  r3  rN   r4  r.   r  r/   get_latest_template_miaplpy  sF   %



 r  c                    s@  t |  ddg} fdd|D d }ddg} fdd|D d }z4d	d  | d
D }tdd |d dD \}}tdd |d dD \}}||||f}	W n   d}	Y z7dd  | d
D }tdd |d dD \}
}tdd |d dD \}}||
||f}W ||	fS    d}Y ||	fS )zRead miaplpy.subset.lalo/yx option from template file into box type
    Return None if not specified.
    (Modified from mintpy.subsets)
    r  r  c                       g | ]}| v r|qS r.   r.   r  Ztmplr.   r/   r   4  rF  z,read_subset_template2box.<locals>.<listcomp>r   r  r  c                    r  r.   r.   r  r  r.   r/   r   6  rF  c                 S   $   g | ]}|  d dddqS [r   ]r  r   r   r.   r.   r/   r   9     $ ,c                 S      g | ]}t | qS r.   rs   r  r   r.   r.   r/   r   :  rF  :c                 S   r  r.   r  r   r.   r.   r/   r   ;  rF  r   Nc                 S   r  r  r  r   r.   r.   r/   r   B  r  c                 S   r  r.   r=  r  r   r.   r.   r/   r   C  rF  c                 S   r  r.   r  r   r.   r.   r/   r   D  rF  )r   r)  rR   sorted)template_filerv   Zkey_laloZkey_yxoptsrx   lat1ry   lon1geo_boxy0y1x0x1pix_boxr.   r  r/   read_subset_template2box-  s*   
    r  c                    sd  dd l m} ddlm} d  d< d  d< t d d \}}z#tt d d d tt d d d g}d	d
 |D }W n   d }Y z' fdd
t D d }tt | d d }t	|
dd dd}W n   t }Y d }	d| v rd}	nd}	|r|	s|d u rd }td td |s|s d dkrt d }
||
}t||d}|d ur||}||}td| td| d }|d urt	|d dd}|	sd| v r||}t||}td| |dv rd|v rddt|d t|d f}|D ]}| vs  | dv r&||  |< q| d< | d<  S )Nr   )subsetr<  r  r  r   rp   r   c                 S   rD  )rp   r   rE  )r   xr.   r.   r/   r   \  r   z#read_subset_box.<locals>.<listcomp>c                    rM  r.   rN  r   inpsDictr.   r/   r   b      r   rq   rl   TFzgWARNING: mintpy.subset.lalo is not supported if 1) no lookup file AND    2) radar/unkonwn coded datasetz	ignore it and continue.r  r   r   )rh   z*input bounding box of interest in lalo: {}z#box to read for datasets in y/x: {}z/box to read for geocoded lookup file in y/x: {}NNoner   r;  )NNONE)Zmintpy.load_data	load_datar   r  r  r:   r   datasetName2templateKeyvaluesr|   rR   r   rv   r   utget_file_listZ'update_box4files_with_inconsistent_sizere   bbox_geo2radarcheck_box_within_data_coveragerI   bbox_radar2geor   r=  )r  Zmldr  r  r  Z
lookupFileZpathKeyrK   r  rw   fnamescoordr  ZatrLutr  r.   r  r/   read_subset_boxL  sl   



r  c                 C   s   t | d}| }W d    n1 sw   Y  dd |D }tt|t| }|| }d }||krCtdtj|  ||fS t	|dt	| k rUtd ||fS t	|}||fS )Nr   c                 S   rD  )r"  r   rE  )r   dater.   r.   r/   r     r   z,update_or_skip_inversion.<locals>.<listcomp>zSAll date exists in file {} with same size as required, no need to update inversion.
   zSNumber of new images is less than 10 --> wait until at least 10 images are acquired)
rF   rG   r   r   r   rI   r'   r(   r]   r;   )Zinverted_date_listZ	slc_datesr   Zinverted_datesZnew_slc_datesZall_date_listZupdated_indexr.   r.   r/   update_or_skip_inversion  s"   
r  c                 C   s   ddl m} tj| d}tj|r2||}|jdd | }| }t	|d t	|d  }nEd
|}|d	
tj| | 7 }t| d
d}	|	 }
t|
}t| t|}|d }|d |d  |d |d   }t|}| }|||fS )Nr   )r  zinputs/slcStack.h5Fr  r;  r   z--template {}z --project_dir {} --work_dir {}load_slc)scriptr<  r   rP  r   )miaplpy.objects.slcStackr  r'   r(   r'  r:  rF   get_date_listr   r=  rI   rT   r   rR   parseread_inps2dictprepare_metadatar  $read_inps_dict2slc_stack_dict_object)r+   r.  r  slc_fileslcObjr^  rg   
num_pixelsscp_argsZParser_LoadSlcZinps_loadSlcr  r<  stackObjr.   r.   r/   read_initial_info  s(   
 
r  c                    sv  ddl m} ddlm} t| }d|d< tjtjt	d}i }| j
D ]}t|}t||d}|| q$| D ]\}}	|	||< q;d| v rP|d |d	< d
  fdd| D }
|
D ]0}| |  }	|dv rt| |  ||< q_|dv rt| |  ||< q_|	r| |  | | < q_d|vs|d dkrd|d< |dd|d< |dd|d< d|vrdd t| j
D }||d |d< t|t|d d |d< |d rtd|d  td|d  |ddrtd|d  |j|d | j|d}tj|d }| jd }tj|st|| tj|d }| jd }tj|s9t|| |S )z.Read input Namespace object info into inpsDictr   )	auto_path)sensorNPLATFORMr  )r-  r  r  miaplpy.load.c                    s$   g | ]}|  r| d  qS r   )r   rR   r   prefixr.   r/   r     r  z"read_inps2dict.<locals>.<listcomp>)r  r  r  autoPath)r  r  r  Fr  r   r  PROJECT_NAMEc                 S   s    g | ]}t j|d kr|qS )r  )r'   r(   r]   r   r.   r.   r/   r     r   zSAR platform/sensor : {}zprocessor: {}r  zAcheck auto path setting for Univ of Miami users for processor: {})r  r+   templatemiaplpy.load.metaFilez/inputs/referencemiaplpy.load.baselineDirz/inputs/baselines)miaplpy.defaultsr  mintpy.objectsr  varsr'   r(   r'  rT   r(  r  r   r)  r7  r   r   rv   r=  r   r   project_name2sensor_namer   r   rI   get_auto_pathr+   r:  rW   copytreer)   )inpsr  r  r  auto_templater  r[   tempr  r  key_listr  Zreference_dirZout_referencebaseline_dirZout_baseliner.   r  r/   r    sj   






r  c                 C   sR  dd l }| d }d|}td td| |dv rHdd |  D D ] }ttt| | dkrEd	|| | }t| t| q%d S |d
krtj	
tj	
| d }tj	| d }tt| d }t|dk ry|d| d  z'|d }	| d }
tj	
| d }dj||||	|
|d}t| t| W d S    Y d S d S )Nr   r  zprep_slc_{}.pyr  z&prepare metadata files for {} products)r   r   r   c                 S   s$   g | ]}| d r|dr|qS )r  r   )r   r   r   r.   r.   r/   r      r  z$prepare_metadata.<locals>.<listcomp>z{} {}r   r   r  r   z No input metadata file found: {}r  r   z.{s} -s {i} -f {f} -m {m} -b {b} -g {g} --force)sr   r   mbg)r  rI   r   rv   r;   r:   r   r'   systemr(   rT   r]   r  r  )r  r  r  script_namer  cmdZslc_dirr  Z
meta_filesZ	meta_filer#  geom_dirr.   r.   r/   r    sH   

r  r   c                 C   s  ddl m} dd l}|dkrXt|  t| d tj}|j}|j}	|t	| }
|	t	| }tj
d|
|d}tj|||d d }t|tj}tj|d |tj
dd	d d }tS td
|  | }|| d  | }|| || || || |  tS )Nr   )Looksr   r   ZENVI)rI   rB  r   )r  ZVRTrI  zMultilooking {0} ...rp   )Zmroipac.looks.Looksr,  isceobjr   r   r  r  RasterXSizeRasterYSizer=  ZTranslateOptionsZ	TranslaterI   createImageloadZsetDownLooksZsetAcrossLooksZsetInputImageZsetOutputFilenameZlooksZoutfilie)r\   rZ   ZrlksZalksZmultilook_toolr,  r-  r  ZxSizeZySizeZoutXSizeZoutYSizeZgdalTranslateOptsZinimgZlkObjr.   r.   r/   	multilook>  s4   



r2  皙?c              
   C   s   | | t | |  }tddd}|t|d dt|   }tt|g}|D ]3}tjdd }dtd|d  td	|d  |d    }	tj	tj
|	d
ddd}	|	|||k< q+|||k }
t|
S )Ng{Gz?r   gMbP?gQ?g)\(?e   r   rP   r  r   )initial)rs   r   arangesqrtrT  r;   ogridsumexpaminamaxr  )N1N2alphaN	distanceslamdaZalpha_cr  nZpvalueZcritical_distancer.   r.   r/   ks_lutf  s    0
rD  c                 C   s0   t | }t || |jd  }t|}|S )z. Estimate Correlation matrix from an ensemble.r   )r   matrixmatmulgetHrU  cov2corr)ZCCGsamZCCGSZcov_matcorr_matrixr.   r.   r/   est_corru  s   
rJ  c              	   C   s>   t ttt| }t|| }t|t|}|S )z= Converts covariance matrix to correlation/coherence matrix. )LApinvr   diagflatr7  diagrF  	transpose)Z
cov_matrixDyrI  r.   r.   r/   rH    s   rH  c                    s  t d t d t d ddlm}m}m} tdd tt D }i }dd |D D ]/}t| }||  v r[t	t

t| | d	 }t|dkr[|||< t d
j||| | d q,d}	|	| vrmt d|	 dS t|| d |	d}i }
| D ]}t|| }||
|< t dj|||d q|t|
 tfddD rd}|d7 }t | | d dvrtj| d d}nd}| d dvrtj| d d}nd}t| }i }||	 D ]}tt|d	d ddd  tj d}d}|dur||krd}|dur||k rd}|rai }tt|D ]:}|| }|| d } |v r3|||< q fd d|| D }t|dkrM|d ||< qt d!|  q| |d"}|| < qt|dkrp||d#}|S d}|S )$z5Read input arguments into dict of slcStackDict objectr  zsearching slcs infozinput data files:r   )slcDatasetNamesslcStackDictslcDictc                 S   r   r.   r   r   r.   r.   r/   r     r   z8read_inps_dict2slc_stack_dict_object.<locals>.<listcomp>c                 S   s   g | ]
}|t  v r|qS r.   )r  rv   r   r.   r.   r/   r     r  rp   z{:<{width}}: {path})rB  r(   r   z(WARNING: No reqired {} data files found!Nr<  )r  dsNameznumber of {:<{width}}: {num})rB  numc                 3   s    | ]	}| d  kV  qdS )r   Nr.   r   )	dsNumListr.   r/   r     r   z7read_inps_dict2slc_stack_dict_object.<locals>.<genexpr>z@WARNING: NOT all types of dataset have the same number of files.z8 -> skip interferograms with missing files and continue.zmiaplpy.load.startDater  z%Y%m%dzmiaplpy.load.endDater   rq   DATETFc                    s   g | ]} |v r|qS r.   r.   r   datesr.   r/   r     rF  z%WARNING: {} file missing for image {})rZ  datasetDict)	pairsDict)r   r  rR  rS  rT  maxr   r  rv   r  r:   r   r;   rI   !skip_files_with_inconsistent_sizer  r   r  strptimer   yyyymmddr|   rR   range)r  rR  rS  rT  maxDigit
dsPathDictrU  r  rJ   ZdsName0Z	dsNumDictZnum_filer   
start_dateend_dateZ
dsNameListr\  ZdsPathZdate_valZinclude_dateZslcPathDictr   ZdsPath1ZdsPath2r  r  r.   )rZ  rW  r/   r    s    

r  c                    s  dd | | D }dd |D }dd |D }d}t dd ||fD rA|du r+d	}n|d
d \}}|t|ks?|t|krAd	}|rt|}	t|}
d}|d7 }|d|	|
7 }|d7 }|d7 }t|  }dd |D }tt|D ]C}|| |	ks|| |
kr|| }t	
| |D ]} fdd| | D }t|dkr| | |d  q|d||| || 7 }qv|d7 }|d7 }|d7 }|d7 }t| | S )z?Skip files by removing the file path from the input dsPathDict.c                 S   s"   g | ]}t |d d d dqS )rp   r   rq   )r|   rR   )r   r[   r.   r.   r/   r     rO  z5skip_files_with_inconsistent_size.<locals>.<listcomp>c                 S      g | ]}t |d  qS )r;  r=  r   r  r.   r.   r/   r     rF  c                 S   rf  )r   rg  rh  r.   r.   r/   r     rF  Fc                 s   s     | ]}t t|d kV  qdS )r   N)r;   r   )r   	size_listr.   r.   r/   r         z4skip_files_with_inconsistent_size.<locals>.<genexpr>NTr      zQ
********************************************************************************zQ
WARNING: NOT all input unwrapped interferograms have the same row/column number!z"
The most common size is: ({}, {})z
------------------------------z)
The following dates have different size:c                 S   r  )rX  r.   rh  r.   r.   r/   r     r   c                    s&   g | ] t  fd dD r qS )c                 3   s     | ]}|d d  v V  qdS )r      Nr.   )r   r_  r   r.   r/   r     rj  z?skip_files_with_inconsistent_size.<locals>.<listcomp>.<genexpr>)rS  )r   rY  rm  r/   r     s    
r   z
	{}	({}, {})z'
Skip loading the interferograms above.z*
Continue to load the rest interferograms.zR
********************************************************************************
)r   r  r  r  rI   r   rv   ra  r;   r   r`  r<   r   )rc  r  rU  r   length_list
width_listZdrop_inconsistent_filesZmax_box_widthZmax_box_lengthZcommon_lengthZcommon_widthr   r[  r^  r   r  r  r.   rY  r/   r^    sJ   


r^  c              
   C   s  |rt ndd }|d |rdd | D }n.|rBt|d}	dd |	j D }W d   n1 s5w   Y  |d	| ntd
|du r|rRtj	|sVt
d|d| i }tj|d }
t|d t|d f}|
dv rt|dS}	t|	jd t|	jd f}|	 D ]8}|	| }t|tjr|jdd |kr|j|j|dd g||< qt|j}||dd< |jt|dg||< qW d   n1 sw   Y  nt|}|d }|D ]}|t|dg||< qtjtj| }tj|st| |d| t| d}|d|  tdd | D }| D ]q}|| d }|| d }|}|dv rAd}t|dkrRd|d |d f}n|}|dj||t|t||d || vrw|j||||d|d }n|| }t|| dkr|| d durt|| d |dd< q*| D ]\}}t||j|< q|dur| D ]\}}|dur||| jd!< |d"|d#| d$|  qW d   n	1 sw   Y  |d%|  | S )&a:  Create HDF5 file with defined metadata and (empty) dataset structure

    Parameters: fname        - str, HDF5 file path
                ds_name_dict - dict, dataset structure definition
                               {dname : [dtype, dshape],
                                dname : [dtype, dshape, None],
                                dname : [dtype, dshape, 1/2/3/4D np.ndarray], #for aux data
                                ...
                               }
                metadata     - dict, metadata
                ds_unit_dict - dict, dataset unit definition
                               {dname : dunit,
                                dname : dunit,
                                ...
                               }
                ref_file     - str, reference file for the data structure
                compression  - str, HDF5 compression type
    Returns:    fname        - str, HDF5 file path

    Example:    layout_hdf5('timeseries_ERA5.h5', ref_file='timeseries.h5')
                layout_hdf5('timeseries_ERA5.5h', ds_name_dict, metadata)

    # structure for ifgramStack
    ds_name_dict = {
        "date"             : [np.dtype('S8'), (num_ifgram, 2)],
        "dropIfgram"       : [np.bool_,       (num_ifgram,)],
        "bperp"            : [np.float32,     (num_ifgram,)],
        "unwrapPhase"      : [np.float32,     (num_ifgram, length, width)],
        "coherence"        : [np.float32,     (num_ifgram, length, width)],
        "connectComponent" : [np.int16,       (num_ifgram, length, width)],
    }

    # structure for geometry
    ds_name_dict = {
        "height"             : [np.float32, (length, width), None],
        "incidenceAngle"     : [np.float32, (length, width), None],
        "slantRangeDistance" : [np.float32, (length, width), None],
    }

    # structure for timeseries
    dates = np.array(date_list, np.string_)
    ds_name_dict = {
        "date"       : [np.dtype("S8"), (num_date,), dates],
        "bperp"      : [np.float32,     (num_date,), pbase],
        "timeseries" : [np.float32,     (num_date, length, width)],
    }
    c                  _   s   d S r&   r.   )argskwargsr.   r.   r/   <lambda>U  s    z#write_layout_hdf5.<locals>.<lambda>r  c                 S      i | ]\}}||qS r.   r.   r   r  r  r.   r.   r/   
<dictcomp>Z      z%write_layout_hdf5.<locals>.<dictcomp>r   c                 S   rs  r.   r.   rt  r.   r.   r/   ru  ]  rv  Nzgrab metadata from ref_file: {}zNo metadata or ref_file found.z"No ds_name_dict or ref_file found!z(grab dataset structure from ref_file: {}r   r;  r   r   r  r   zcrerate directory: {}az create HDF5 file: {} with w modec                 S   r   r.   r   r   r.   r.   r/   r     r   z%write_layout_hdf5.<locals>.<listcomp>r   )r   lzfrP  r   zOcreate dataset  : {d:<{w}} of {t:<25} in size of {s:<20} with compression = {c})r_  rQ   tr$  r4  T)rU  maxshaper   chunksr  r   zadd /<z attribute: UNIT = zclose  HDF5 file: {}) r   r   r   r   r   rI   r   r'   r(   rS   r   r   r=  rv   r   r   rU  r   r   tupler   rR  rT   r)   r^   makedirsr]  r;   r   create_datasetr   array)r[   ds_name_dictrg   ds_unit_dictref_filer  r   vprintmetarL   r   shape2dshape2d_origr  r  ds_shapeds_namesds_dtypeds_namefdirr   	max_digitr  
data_shapeds_comp	max_shaper  r.   r.   r/   write_layout_hdf5%  s   0



$

4r  )NTN)Nr  N)NNT)NN)NNFr  )Nr   )r   r   )r3  )Nr   )NNNNNT)<r'   rW   r:   mintpy.objects.coordr   r   osgeor   r  r*  numpyr   miaplpy.objects.arg_parserr   mintpy.utilsr   r   r   r  r  r   r	   r
   r   r   r   r   r   r   r   r  r%   re   rE   r|   r7  rV   r>  r?  rR  r  r  r  r   r  r  r  r  r  r  r  r  r  r2  rD  rJ  rH  r  r^  r  r.   r.   r.   r/   <module>   s   0	
e*
 }

O
/
K '
N

%2SG
%
(

f2