U
    bbܒ                     @   s&  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlmZmZmZmZmZmZ ejZdZddd	d
dgZe	dZdZdZdZdd Zd(ddZ dd Z!d)ddZ"dd Z#dd Z$dd Z%d*ddZ&d d! Z'd+d"d#Z(d,d$d%Z)e*d&kr"e)ej+d'd  dS )-    N)linalg)get_template_content)
timeseriesgiantTimeseriesHDFEOScluster)	arg_groupptime	time_funcreadfile	writefileutilsmintpy.velocity.	startDateendDateexcludeDate	bootstrapbootstrapCountvelocitya  references:
  Fattahi, H., and F. Amelung (2015), InSAR bias and uncertainty due to the systematic and stochastic
  tropospheric delay, Journal of Geophysical Research: Solid Earth, 120(12), 8758-8773, doi:10.1002/2015JB012419.

  Efron, B., and R. Tibshirani (1986), Bootstrap methods for standard errors, confidence intervals,
  and other measures of statistical accuracy, Statistical science, 54-75, doi:10.1214/ss/1177013815.
a  example:
  timeseries2velocity.py  timeseries_ERA5_demErr.h5
  timeseries2velocity.py  timeseries_ERA5_demErr_ramp.h5  -t KyushuT73F2980_2990AlosD.template
  timeseries2velocity.py  timeseries.h5  --start-date 20080201  --end-date 20100508
  timeseries2velocity.py  timeseries.h5  --exclude exclude_date.txt

  timeseries2velocity.py  LS-PARAMS.h5
  timeseries2velocity.py  NSBAS-PARAMS.h5
  timeseries2velocity.py  TS-PARAMS.h5

  # bootstrapping for STD calculation
  timeseries2velocity.py timeseries_ERA5_demErr.h5 --bootstrap

  # complex time functions
  timeseries2velocity.py timeseries_ERA5_ramp_demErr.h5 --poly 3 --period 1 0.5 --step 20170910
  timeseries2velocity.py timeseries_ERA5_demErr.h5      --poly 1 --exp 20170910 90
  timeseries2velocity.py timeseries_ERA5_demErr.h5      --poly 1 --log 20170910 60.4
  timeseries2velocity.py timeseries_ERA5_demErr.h5      --poly 1 --log 20170910 60.4 200 --log 20171026 200.7
z-exclude_date.txt:
20040502
20060708
20090103
c                  C   s~  t jdt jtd t d t d} | jddd | jddd	d
d | jdddd | jddddd | jddddd | jdddtddd | jddd tdd!d | jd"d#d$d%d& | 	d'}|jd(d)d*d+d |jd,d-d.d/d |jd0d1d2d3g d4t
 d5 | 	d6d7}|jd8d9d:dd;d |jd<d=d>td?d@dA t| } | 	dBdC}|jdDdEdFddGd |jdHdIdJdKdLdM t| } | S )NNz4Estimate velocity / time functions from time-series.
)descriptionformatter_classepilogtimeseries_filez(Time series file for velocity inversion.)helpz
--templatez-ttemplate_fileztemplate file with options)destr   z--ts-cov-filets_cov_filez:Time-series (co)variance file for velocity STD calculationz-oz--outputoutfilezoutput file namez--updateupdate_mode
store_truezEnable update mode, and skip estimation if:
1) output velocity file already exists, readable and newer than input file
2) all configuration parameters are the same.)r   actionr   z
--ref-laloref_lalo)LATLON   z-Change referene point LAT LON for estimation.)r   metavartypenargsr   z--ref-yxref_yx)YXz)Change referene point Y X for estimation.z
--ref-dateref_dateDATEz%Change reference date for estimation.)r   r&   r   zdates of interestz--start-datez-sr   z"start date for velocity estimationz
--end-datez-er   z end date for velocity estimationz	--excludez--exr   +zsdate(s) not included in velocity estimation, i.e.:
--exclude 20040502 20060708 20090103
--exclude exclude_date.txt
)r   r(   defaultr   Zbootstrappingz3estimating the mean / STD of the velocity estimatorz--bootstrapz--bootstrappingr   zLEnable bootstrapping to estimate the mean and STD of the velocity estimator.z--bcz--bootstrap-countr   i  z>number of iterations for bootstrapping (default: %(default)s).)r   r'   r/   r   zResidual filez4Save residual displacement time-series to HDF5 file.z
--save-resz--save_residualsave_resz8Save the residual displacement time-series to HDF5 file.z
--res-filez--residual-fileres_fileztimeseriesResidual.h5zJOutput file name for the residual time-series file (default: %(default)s).)r   r/   r   )argparseArgumentParserRawTextHelpFormatterTEMPLATE	REFERENCEEXAMPLEadd_argumentfloatintadd_argument_groupDROP_DATE_TXTr   add_timefunc_argumentadd_memory_argument)parserdater   resid rB   P/home/centos/operations/rsmas_insar/sources/MintPy/mintpy/timeseries2velocity.pycreate_parserJ   sj    






rD   c                 C   sR  t  }|j| d}t|jd |_|jdkr>td|j|jrd|j	dkrdd|_t
d t
d |jrt
d	 |jdks|js|js|js|jrtd
|jrt|j|}t|}|jrNt|j}t|}||jd |jd dd \}}|dk	rN|dk	rN||g|_t
d|jd |jd  t
d|jd |jd  |S )zCommand line parser.)args	FILE_TYPE)r   r   r   z!input file is {}, NOT timeseries!   FzIbootstrap-count should be larger than 1, otherwise it does not make sensez/turn OFF bootstrapping and continue without it.zbootstrapping is turned ON.zMbootstrapping currently support polynomial ONLY and ONLY with the order of 1!r   Nr%   z-input reference point in (lat, lon): ({}, {})z)corresponding   point in (y, x): ({}, {}))rD   
parse_argsr   read_attributer   key	Exceptionformatr   r   print
polynomialperiodicstepexplog
ValueErrorr   read_template2inpsinit_exp_log_dictsr"   ut
coordinate	geo2radarr)   )iargsr?   inpsatrcoordref_yref_xrB   rB   rC   cmd_line_parse   s2    
"
$
r_   c                 C   sV  t  | _| jr| jD ]}|d |dd  }}t|dkrt|dkrbt|t | j|< qd	|}|d7 }|d7 }|d7 }|d	7 }t
|qt
d
	|qt  | _| jrR| jD ]}|d |dd  }}t|dkrBt|dkrt|t | j|< n2d	|}|d7 }|d7 }|d7 }|d	7 }t
|qt
d
	|q| S )zqInitialize the dictionaries of exp and log funcs
    By trarnslating inps.exp/log into inps.expDict/logDict.
    r   rG   N   z!NO characteristic time found: {}
zCone or more characteristic time(s) are required for each onset datez for the exp function, e.g.:
z--exp 20181026 60 OR
zA--exp 20161231 80.5 200  # append as many char_times as you like!z.input onset time is NOT in YYYYMMDD format: {}z for the log function, e.g.:
)dictexpDictrQ   lennparrayastyper9   tolistrL   rS   logDictrR   )rZ   Zexp_listZ
onset_timeZ
char_timesmsgZlog_listrB   rB   rC   rU      s:    





rU   c                    s  |s
t  }t|}tdtj|   t|jt	
d  fddt| D }|D ]} |  }|dkr|||< |r`|dkrt|||< q`|dkr|dd	d
d	dd}t| ||< q`|dkr`t|||< q`d}| kr| rt| |_|S )z.Read input template file into inps.excludeDatez!read options from template file: r   c                    s    g | ]} |   kr|qS rB   )keys.0iprefixtemplaterB   rC   
<listcomp>   s      z&read_template2inps.<locals>.<listcomp>)r   )r   r   r   [ ], )r   zmintpy.compute.maxMemory)r_   varsrM   ospathbasenamer   read_templater   rV   check_template_auto_valuelistrj   r	   yyyymmddreplacesplitr:   r9   	maxMemory)r   rZ   iDictkeyListrJ   valuerB   rn   rC   rT      s0    
rT   c                    s   t d d}tjjs0d}t dj nZt dj tjj}tjj}||krzd}t dj nt dj |dkrt	j t
 fdd	tD rd}t d
t nt dt t d| |S )Nzupdate mode: ONskiprunz1) output file {} NOT found.z!1) output file {} already exists.z02) output file is NOT newer than input file: {}.z,2) output file is newer than input file: {}.c                 3   s.   | ]&}t t|  t| d kV  qdS )NoneN)strrx   get
key_prefix)rl   rJ   r[   rZ   rB   rC   	<genexpr>  s     zrun_or_skip.<locals>.<genexpr>z93) NOT all key configuration parameters are the same: {}.z53) all key configuration parameters are the same: {}.zrun or skip: {}.)rM   ry   rz   isfiler   rL   getmtimer   r   rI   any
configKeys)rZ   flagtitorB   r   rC   run_or_skip   s(    r   c                 C   s$  t |}g }|t jt| j|d7 }|r:tdt|  | jrtd| j  t t | j}t	t
|D ]6}|| }|| |k rl||krltd|  || ql| jrtd| j  t t | j}t	t
|D ]6}|| }|| |kr||krtd|  || qttt|}|S )N)date_list_allzexclude date:zstart date: z  remove date: z
end date: )r	   yyyymmdd2yearsread_date_listr~   r   rM   r   r   r   rangerc   appendr   sortedset)rZ   dateListAllZyy_list_all
exDateListZyy_minrm   r@   Zyy_maxrB   rB   rC   read_exclude_date  s.    
r   c           
         s@   j dkrt j}n* j dkr,t j}n j dkr@t j}|  t |j _t	j
 jdrt j }t j\}}tj|dddk}d|||d < t|dkrtd	t|   jt||  7  _ttt j _ fd
d|jD  _t j _ jd  _ jd  _td td|j|j td t jt|jkr|td ntd j j td tj fdd|jD tjd _  j!s<t	j
"t	j
 jd }d} j dkrt	j
 j#dd }|| }n |dkr.|#dd }	||	 }|d7 }| _! S )z~Read dates used in the estimation and its related info.
    Parameters: inps - Namespace
    Returns:    inps - Namespace
    r   r   r   timeseriesRg)rG   r%   axisr   REF_DATEz$number of empty dates to exclude: {}c                    s   g | ]}| j kr|qS rB   rr   rk   rZ   rB   rC   rq   X  s     
 z"read_date_info.<locals>.<listcomp>z2--------------------------------------------------zdates from input file: {}
{}z)using all dates to calculate the velocityz*dates used to estimate the velocity: {}
{}c                    s   g | ]}| j kqS rB   rr   rk   r   rB   rC   rq   f  s     dtyper   ZPARAMS)r   ZtimeseriesAzz.h5)$rJ   r   r   r   r   openr   dateListr   ry   rz   r{   
startswithget_date_listr   readrd   nansumindexsumrM   rL   re   rg   r   r~   r   rc   numDater   r   bool_dropDater   splitextr   )
rZ   Ztsobj	date_listdatar[   r   fbaseoutnamero   suffixrB   r   rC   read_date_info>  sR    




"

r   c                 C   s  |s
| j }|d |d  }}t|}t|}| jr|| jD ]>}t|}||  k r^|k s<n td| d| d| q<| jr| j D ]*}t|}	|	|krtd| d| q| jr| j D ]*}t|}	|	|krtd| d| qt }
| j	|
d	< | j
|
d
< | j|
d< | j|
d< | j|
d< td |
 D ]\}}td|| q<d	|
 krntd|
d	 d t|
d
 d  t|
d  tdd |
d  D  tdd |
d  D  }|
|fS )zget model info from inpsr   r   zinput step date "z" exceed date list min/max: z, zinput exp onset date "z" >= the last date: zinput log onset date "rN   rO   rP   rQ   rR   zEestimate deformation model with the following assumed time functions:z    {:<10} : {}z7linear/polynomial model is NOT included! Are you sure?!rG   r%   c                 S   s   g | ]\}}t |qS rB   rc   rl   rJ   valrB   rB   rC   rq     s     z#read_inps2model.<locals>.<listcomp>c                 S   s   g | ]\}}t |qS rB   r   r   rB   rB   rC   rq     s     )r   r	   r   rP   rS   rb   rj   rh   ra   rN   rO   rM   itemsrL   rc   r   )rZ   r   dmindmaxyminymaxZd_stepy_stepZd_onsetZy_onsetmodelrJ   r   	num_paramrB   rB   rC   read_inps2modelx  sT    












r   c           5   	   C   sX	  t | j}t|d t|d  }}| j}t| j}|dd}d|	 kr~| j
s~| jd | _
td td| jd  t| \}}t|}	d|	d	< d
|	d< | jd |	d< | jd |	d< d| jd | jd |	d< | jr| jd |	d< | jd |	d< | j
r| j
|	d< tdt tD ]}
tt| |
 |	t|
 < q t|||fddd  \}}tj| j|	||d | jrt|}dD ]}
|
|	 kr||
 qt| jd }td| |ftj| jtjdgtj|||fd gd}tj| j||d ||d  d | | d }| jr<|| j | | | d 7 }tt!|d | j"d  }t#j$dd||f|d d!d"}t%|D ]\}}|d |d  }|d |d  }|| }|dkrtd#|d | td$| td%| tj&||ft'd}tj&||ft'd}td&| j t j(| j|d'd }| j
rtd(| j
 | j)| j
}|t*||d d d d f |j+d ddf8 }| jrtd)| jd | jd  | jd | jd | jd d | jd d f}t j(| j|d'd }|t*|,|j+d ddd|j+d |j+d f8 }|| j-d d d d f ,| jd}|	d d*krP|d+9 }d }| j.r<td,| j. d- t j(| j.|d'd }t|j+dkr| j|j+d k r|| j-d d d d d d f }|d d | j-d d d d f }|,| j| jd}nHt|j+dkr<| j|j+d k r.|| j-d d d d f }|,| jd}td. tj/|dd/}t0t1| |d0k}~|d d |f }tt2|} t3|d }!td1| || | d2  | dkrqztd3 | jrtd4| j  tj45 }"tj&| j || ft'd}#t6j7| j d5}$t8| j D ]h}|"j9| j| jd!d6}%|%:  t;j<|||% = ||% |d7d |#|< |$j>|d d8|d | j d9 q|$?  |#j@dd/,|d|d d |f< |#jAdd/,|d|d d |f< ~#t;jB| j|| j
|d:}&n|t;j<|| j||d7\}&|d d |f< }'|d k	rt|j+dkr*d!nd;}(d<})|)|(r>d=nd>7 })t|) tCD|&}*t6j7| d5}$t8| D ]}|!| }+|(r|d d d d |+f ntE|d d |+f },tjCF|*|,|*jGg}-tHtE|-|d d |+f< |$j>|d d?d@|d | dA qj|$?  n\tdB tCItJ|&jG|&}.|',dd||  }/tHtJtE|.,dd|/|d d |f< |d |d |d |d g}0t||||dCd }1|1K D ]&\}2}3tjL| j|3,|||2|0dD q| jrzd||d |d |d |d g}0tjM||| ftjdtjN }4|tJ|&|d d |f  |4d d |f< tjL| j|4,|||dE|0dD qz| jS )FNLENGTHWIDTHCENTER_LINE_UTCr   r   zHWARNING: No REF_DATE found in time-series file or input in command line.z#  Set "--ref-date {}" and continue.r   rF   m/yearUNIT
START_DATEr   END_DATEz{}_{}DATE12REF_YrG   REF_Xz3add/update the following configuration metadata:
{})ds_shape)metadatads_name_dictds_unit_dict)r   Sr   )r@   r   )r   r   r%         i   @yT)box	num_split	dimension	print_msgz5
------- processing patch {} out of {} --------------zbox width:  {}zbox length: {}zreading data from file {} ...)r   zreferecing to date: {}z%referencing to point (y, x): ({}, {})mmgMbP?z0reading time-series covariance matrix from file z ...z3skip pixels with zero/nan value in all acquisitionsr   g        z2number of pixels to invert: {} out of {} ({:.1f}%)d   z.estimating time functions via linalg.lstsq ...zEestimating time function STD with bootstrap resampling ({} times) ...)maxValue)sizer   )r   r   dis_tssecondsziteration {} / {})r   )r   r,   r   Fz/estimating time function STD from time-serries zcovariance pixel-by-pixel ...zvariance pixel-by-pixel ...   z{}/{} pixels)everyr   zBestimating time function STD from time-series fitting residual ...)mask)r   datasetNameblockr   )Or   rI   r   r:   r   rd   re   r   r   rj   r,   rM   rL   r   ra   r)   r   r   rx   r   model2hdf5_datasetr   layout_hdf5r   r0   poprc   r   string_float32r1   r   r   ceilr   r   split_box2sub_boxes	enumeratezerosdataTyper   r   tileshapereshaper   r   nanmeanmultiplyisnanr   whererandomdefault_rngr	   progressBarr   choicesortr
   estimate_time_funcrg   updateclosemeanstdget_design_matrix4time_funcr   pinvdiag	multi_dotTsqrtinvdotr   write_hdf5_blockonesnan)5rZ   r[   lengthwidthnum_datedatesr   r   r   ZatrVrJ   r   r   ZatrRZdate_lenZ	memoryAllnum_boxbox_listrm   r   box_widbox_len	num_pixelmm_stdts_dataref_indref_boxref_valts_covZts_stackr   num_pixel2invidx_pixel2invrngZm_bootprog_barZboot_indGe2Z
covar_flagri   Gplusidxts_coviZm_covZG_invZm_varr   ds_dictds_namer   ts_resrB   rB   rC   run_timeseries2time_func  sR   
$
.,"  



$  

2$
,	

 (
r)  c                    s  | d }t | d }t | d }tdd | d  D }i }	i }
i }|dk	r|jdkrb|jd nd}|d	|}|d	|}|dkrtj|tjd
}n|	 }t
d|d D ]}|dkrd}d}n&|dkrd}d}nd|}d|}|dk	r ||ddf |	|< ||ddf |	|d < t|dg|
|< |||< t|dg|
|d < |||d < q|d }t
|D ]N}| d |  ddg} dkrdd |D }n, dkrdd |D }n fdd|D }|dk	rx||d|  ddf }||d|  d ddf }t|d |d  }tj|td
}t|| dksVt|| ||  ||< t|||gD ]\}}||	|< qdt|dg|
|d < d||d < t|dg|
|d < d||d < qf|d d|  }t
|D ]}d| d | }|dk	r$||| ddf |	|< ||| ddf |	|d < t|dg|
|< d||< t|dg|
|d < d||d < q|d d|  | }d}| d  D ]}| d | D ]}d||}|dk	r||| ddf |	|< ||| ddf |	|d < t|dg|
|< d||< t|dg|
|d < d||d < |d7 }qq|d d|  | | }d}| d  D ]}| d | D ]}d||}|dk	r||| ddf |	|< ||| ddf |	|d < t|dg|
|< d||< t|dg|
|d < d||d < |d7 }qZqJ|	|
|fS ) a#  Prepare the estimated model parameters into a list of dicts for HDF5 dataset writing.
    Parameters: model        - dict,
                m            - 2D np.ndarray in (num_param, num_pixel) where num_pixel = 1 or length * width
                m_std        - 2D np.ndarray in (num_param, num_pixel) where num_pixel = 1 or length * width
                mask         - 1D np.ndarray in (num_pixel), mask of valid pixels
                ds_shape     - tuple of 2 int in (length, width)
    Returns:    ds_dict      - dict, dictionary of dataset values,     input for writefile.write_hdf5_block()
                ds_name_dict - dict, dictionary of dataset initiation, input for writefile.layout_hdf5()
                ds_unit_dict - dict, dictionary of dataset unit,       input for writefile.layout_hdf5()
    Examples:   # read input model parameters into dict
                model = read_inps2model(inps, date_list=inps.date_list)
                # for time series cube
                ds_name_dict, ds_name_dict = model2hdf5_dataset(model, ds_shape=(200,300))[1:]
                ds_dict = model2hdf5_dataset(model, m, m_std, mask=mask)[0]
                # for time series point
                ds_unit_dict = model2hdf5_dataset(model)[2]
                ds_dict = model2hdf5_dataset(model, m, m_std)[0]
    rN   rO   rP   c                 S   s   g | ]\}}t |qS rB   r   r   rB   rB   rC   rq     s     z&model2hdf5_dataset.<locals>.<listcomp>rQ   NrG   r   r   r   r   r%   accelerationzm/year^2zpoly{}z	m/year^{}StdZ	AmplitudePhasec                 S   s   g | ]}d | qS )ZannualrB   rl   xrB   rB   rC   rq     s     g      ?c                 S   s   g | ]}d | qS )Z
semiAnnualrB   r-  rB   rB   rC   rq     s     c                    s   g | ]}d   | qS )ZperiodYrB   r-  periodrB   rC   rq     s     r   r  radianzstep{}z
exp{}Tau{}rR   z
log{}Tau{})rc   r   r   ndimr   r   rd   r  r   flattenr   rL   r   r  r   allarctanziprj   )r   r  r  r   r   poly_deg
num_periodnum_stepnum_expr&  r   r   r  rm   dsNameunitp0ZdsNameSuffixesdsNamesZcoef_cosZcoef_sinZ
period_ampZ
period_phar   	exp_onsetexp_tau	log_onsetlog_taurB   r/  rC   r     s    








r   c                 C   sd   t | }t }t|}|jr0t|dkr0|jS t| tt | d\}}td	|| |jS )Nr   <   z'time used: {:02.0f} mins {:02.1f} secs.)
r_   timer   r   r   r   r)  divmodrM   rL   )rY   rZ   
start_timer  srB   rB   rC   maint  s    rH  __main__rG   )N)N)N)NNNN)N),ry   sysrD  r2   numpyrd   scipyr   mintpy.defaults.templater   mintpy.objectsr   r   r   r   mintpy.utilsr   r	   r
   r   r   r   rV   r   r   r   r   r5   r6   r7   r<   rD   r_   rU   rT   r   r   r   r   r)  r   rH  __name__argvrB   rB   rB   rC   <module>   sH    
?
&-
 !!:
7  "
 &

