o
    gRed                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlZd dlZd dlmZmZmZmZ d dlmZ d dlZdZdZdd	 Zd=d
dZdd Zd=ddZdd Zdd Z dd Z!dd Z"dd Z#dd Z$dd Z%dd Z&d d! Z'd"d# Z(d$d% Z)d&d' Z*d(d) Z+d*d+ Z,d,d- Z-d.d/ Z.d0d1 Z/d2d3 Z0d4d5 Z1d6d7 Z2d8d9 Z3d=d:d;Z4e5d<kre4  dS dS )>    N)KNeighborsClassifier)partial)readfileptime	writefileutils)
timeseriesa  
Please Note:
1. Four types of data are supported: velocity, timeseries, maskPS and maskTempCoh
2. This script concatenates two datasets together. The input files are object datasets, their corresponding geometryRadar.h5 files, and the whole region geometryRadar.h5 file processed with 1:1 looks
3. If a batch concatenation needed, please use the concatnate_patches.py script.
4. For timeseries, datasets should have same reference date

a  example:

    concatenate_radarGeo.py miaplpy_NE/velocity_msk.h5  miaplpy_NNE/velocity_msk.h5 --geo-file1 miaplpy_NE/inputs/geometryRadar.h5 --geo-file2 miaplpy_NNE/intpus/geometryRadar.h5 --geo-full ./inputs/geometryRadar.h5 --geo-write --outdir miaplpyBig

    concatenate_radarGeo.py miaplpy_NE/velocity_msk.h5  miaplpy_NNE/velocity_msk.h5 --geo-full ./inputs/geometryRadar.h5 --outdir miaplpyBig
    
    concatenate_radarGeo.py miaplpy_NE/timeseries_msk.h5 miaplpy_NNE/timeseries_msk.h5 --geo-file1 miaplpy_NE/inputs/geometryRadar.h5 --geo-file2 miaplpy_NNE/inputs/geometryRadar.h5 --geo-full ./inputs/geometryRadar.h5  --out-suffix NE_NNE --outdir ./miaplpyBig/

    concatenate_radarGeo.py miaplpy_NE/maskPS.h5  miaplpy_NNE/maskPS.h5 --geo-file1 miaplpy_NE/inputs/geometryRadar.h5 --geo-file2 miaplpy_NNE/intpus/geometryRadar.h5 --geo-full ./inputs/geometryRadar.h5  --out-suffix NE_NNE --outdir miaplpyBig

    concatenate_radarGeo.py miaplpy_NE/maskTempCoh.h5  miaplpy_NNE/maskTempCoh.h5 --geo-file1 miaplpy_NE/inputs/geometryRadar.h5 --geo-file2 miaplpy_NNE/intpus/geometryRadar.h5 --geo-full ./inputs/geometryRadar.h5  --out-suffix NE_NNE --outdir miaplpyBig
c                  C   s   t jdt jtd t d} | jddtdd | jdd	td
gdd | jdd	td
gdd | jdd	tdd | jddddd | jdd	dgdd | jddd	g dd | jddddd | S ) NzConcatenate miaplpy patches
)descriptionformatter_classepilogpatch_files+z.two displacement datasets to be concatenated 
)nargstypehelpz--geo-file1   defaultzgeometryRadar file of dataset1.The default means using geometryRadar.h5 existing in the corresponding patch_files inputs folder
)r   r   r   r   z--geo-file2zgeometryRadar file of dataset2.The default means using geometryRadar.h5 existing in the corresponding patch_files inputs folder
z
--geo-fullz>Whole region geometryRadar.h5 file processed with 1:1 looks. 
z--geo-write
store_trueFz:whether write the concatenated geometryRadar.h5 results. 
)actionr   r   z--out-suffix z-suffix of output name of concatenated data. 
)r   r   r   z--outdiroutdirz2output directory to store the concatenated results)destr   r   r   z--listzHlist the files used for concatenation and check the order of latitude. 
)argparseArgumentParserRawTextHelpFormatterNOTEEXAMPLEadd_argumentstr)parser r!   P/home/exouser/operations/rsmas_insar/tools/MimtPy/mimtpy/concatenate_radarGeo.pycreate_parser/   s&   
r#   c                 C   s   t  }|j| d}|S )N)args)r#   
parse_args)iargsr    inpsr!   r!   r"   cmd_line_parseK   s   r(   c                 C   s   | j }ttj|dS )N)newshape)shaper   npreshape)xoriginal_shaper!   r!   r"   flatten_transQ   s   r/   c                 C   s   |d u r| d d }|d d }nF|dks!|dks!|dks!|dkr.| d d }|d d }n)|dks:|dks:|dkrG| d d	 }|d d	 }n|d
krW| d	 d }|d	 d }||k}||k}|| }	t |	dk}
|
d d |
d d fS )Nr   r                        T)r+   where)lat_sublon_sublatlonref_flag
lat_corner
lon_cornerlat_flaglon_flagflagpos_flagr!   r!   r"   geo_position_backupU   s"    rD   c                 C   s   ddg}ddg}t jdtdt j }d}|D ]>}|D ]9}	| | |	 }
|| |	 }||
k}||k}|| }t |dk}|d d || d< |d d || d< |d7 }qq|S )Nr   r6   )r0   r3   dtypeTr   )r+   onesintnanr8   )r9   r:   r;   r<   row_listcol_listrow_col_matrixnumrowcolr>   r?   r@   rA   rB   posr!   r!   r"   get_positionk   s"   
rQ   c                 C   st   t t| d |d g}t t| d |d g}t t| d |d g}t t| d |d g}||||fS )Nr   r   )r6   r   r   r   )r6   r   )rH   r+   minmax)rc_refrc_affrow_join_upperrow_join_lowercol_join_leftcol_join_rightr!   r!   r"   design_joined_matrix   s
   r\   c                 C   s   t | d }|| S Nr3   )r+   sin)thetavr!   r!   r"   haversin   s   ra   c           	      C   s|   d}t | } t |}t |}t |}|| }||  }t|t| t| t|  }d| tt| }|S )Ni  r3   )mathradiansra   r+   cosr^   sqrt)	lat1lon1lat2lon2radiusdlondlathdisr!   r!   r"   distance2points   s   



$ro   c                 C   s   t | }t | }t |}t |}t |}t |}	t |}
t |}t||}t||
}t||	}t||}||||fS N)r+   rT   rU   )lat1_flattenlon1_flattenlat2_flattenlon2_flattenlat1_minlat1_maxlon1_minlon1_maxlat2_minlat2_maxlon2_minlon2_maxover_lat_minover_lon_minover_lat_maxover_lon_maxr!   r!   r"   overlay_lalo   s   











r   c           	      C   s   t | d d df |k | d d df |k@ }t | d d df |k | d d df |k@ }t ||}t|}td|  |S )Nr   r   z:The total number of PS located at the overlay region is %d)r+   r8   intersect1dlenprint)	latlonr}   r~   r   r   flag_latflag_lonrB   PS_numr!   r!   r"   
PS_overlay   s   ..r   c                 C   s&  ||| d}t |}|||d}t |}	|	jd d ddgf }
dgt|
 }|jd d ddgf }tdddd d}||
| |j|ddd	\}}tdgg}|	 D ]5\}}|	j||  }|| d d
k rt |j
}|j|df }|d || d }|| }t||}qW|dd  S )N)r<   r;   velr   r   	ball_treec                 S   s   t g | |R  S rp   )ro   )s1s2r!   r!   r"   <lambda>   s    z)calculate_offset_matrix.<locals>.<lambda>)n_neighbors	algorithmmetricT)r   return_distanceg~jtx?r   )pd	DataFrameilocr   r   fit
kneighborsr+   arrayiterrowsTlocgetappend)vel_reflat_reflon_refvel_afflat_afflon_afffind_PSfinddata_PSdatadata_fityfind_xknndistancepointoffsetirN   tmpfind_svel_ref_valuevel_aff_value	vel_deltar!   r!   r"   calculate_offset_matrix   s0   

r   c                 C   s  t t t |gt t |gf}t t t |gt t |gf}t||||\}}	}
}t|||	|
|}t|||	|
|}| | }t | }|| }t | }t|| }td|  t|| }td|  ||kr|| }|d d df | | }|d d df | | }|| }|d d df | | }|d d df | | }t	||||||}nE|| }|d d df | | }|d d df | | }|| }|d d df | | }|d d df | | }t	||||||}|d9 }t 
|}td|  || }|S )NzGThe Nan PS point of reference image located in the overlay region is %dzGThe Nan PS point of affiliate image located in the overlay region is %dr   r   r6   zThe overlay offset is %f)r+   hstack	transposer   r   r   isnanr   r   r   	nanmedian)data1_flattendata2_flattenrq   rr   rs   rt   latlon1latlon2r}   r~   r   r   PS_flag1PS_flag2	data1_tmpmask1	data2_tmpmask2data1_overlay_numdata2_overlay_numdata_refr   r   data_affr   r   r   overlay_offsetdata2_flatten_adjustr!   r!   r"   concatenate_process   sB   **
r   c                 C   s  dd }t ||\}}}	}
t|| d |
|	 d ftj }tt|d |d  }tt|d |d  }td|  |dksG|dkrtd|jd	 |jd  | |d |d f }|d	| jd	 | d	| jd | f }t	|| }td
|  | |d	| jd	 d	| jd f< || ||d |d f< ||||||| jd	 || jd f< |dkrd	|t
|tjk< |S |dks|dkr=td|jd	 |jd  | |d d	|jd | f }|d	| jd	 | |d f }t	|| }| |d	| jd	 |d f< || ||d d	|jd f< ||||||| jd	 ||jd f< |dkr;d	|t
|tjk< |S |dkrtd|jd	 |jd  | d	|jd	 | |||jd  f }||d d d f }t	|| }| ||d d	| jd f< || |d	|jd	 |||jd  f< |||||||jd	 |||jd  f< |dkrd	|t
|tjk< |S |dkr=td|jd	 |jd  | |d |||jd  f }|d	| jd	 | d d f }t	|| }| |d	| jd	 d	| jd f< || ||d |||jd  f< ||||||| jd	 |||jd  f< |dkr;d	|t
|tjk< |S |dkrtd|jd	 |jd  | |||jd	  d	|jd | f }|d d |d f }t	|| }| |d	| jd	 |d f< || ||||jd	  d	|jd f< ||||||||jd	  ||jd f< |dkrd	|t
|tjk< |S |dkr@td|jd	 |jd  | |||jd	  |d f }|d d d	| jd | f }t	|| }| |d	| jd	 d	| jd f< || ||||jd	  |||jd  f< ||||||||jd	  || jd f< |dkr@d	|t
|tjk< |S )Nc                 S   sH   | | | d }|t |  | |t | < | t | |t |< |S r]   )r+   r   )overlap_refoverlap_affr   tempr!   r!   r"   
common_cal  s   z"concatenate_2D.<locals>.common_calr   rR   rS   zThe ref_flag is %dr0   z"Full the concatenated data: {}, {}r   zThe offset is %fmskr3   r4   r7   r1   r5   r2   )r\   r+   rG   rI   rH   absr   formatr*   r   r8   )val_refval_affrV   rW   r=   	data_typer   row_join_startrow_join_endcol_join_startcol_join_endval_joinrow_a_rcol_a_rr   r   r   r!   r!   r"   concatenate_2D  s   $( (:(
.
((,
#
 ",

((,

 ,,
r   c
                 C   s   t |||||||	\}
}}}}}}}| j|
 }| j| }td tj|dd\}}| }td tj|dd\}}| }d}t|||||	|}|}|jd |d< |jd |d	< ||fS )
NRead the reference datasetvelocitydatasetNameRead the affiliate datasetr   r   LENGTHr   WIDTH)appoint_refr   r   r   readflattenr   r*   )r'   rq   rr   rs   rt   unflatten_trans1unflatten_trans2rc1rc2r=   ref_Noaff_NorV   rW   lat_ref_flattenlon_ref_flattenlat_aff_flattenlon_aff_flattenr   r   r   vel_ref_atrvel_ref_flattenr   vel_aff_atrvel_aff_flattenr   
vel_joinedvel_atrr!   r!   r"   concatenate_velf  s   $


r   c
           :      C   s.  t |||||||	\}
}}}}}}}| j|
 }| j| }td tj|dd\}}td tj|dd\}}t|d}|d }t| }t|d}|d }t| }|j	d }|j	dd	 \}} |j	d }!|j	dd	 \}"}#t
j||||!||\}$}%}&}'t|%}(t||\})}*}+},|*|) d }-|,|+ d }.t }/tj|(|-|.ftd
}0d}1t|%|&D ]<\}2}3td|2  tj||2dd }4|4 }5tj||3dd }6|6 }7d}8t|4|6|||	|8|0|1d d d d f< |1d7 }1qtj|'td|/d< tj|$tjd|/d< |0|/d< |}9|0j	d |9d< |0j	d |9d< |/|9|$fS )Nr   r   r   r   rz/bperpr   r   r4   )r*   rF   z$Process displacement data of date %stsrE   bperpdater   r3   r   )r   r   r   r   r   h5pyFiler   get_date_listr*   mimtpyconcatenate_offset
date_matchr   r\   dictr+   emptyfloatzipr   r   r   string_):r'   rq   rr   rs   rt   r   r   r   r   r=   r   r   rV   rW   r   r   r   r   r   r   ts_ref
ts_ref_atrts_aff
ts2_affatrbperp_date_ref	bperp_refdateList_refbperp_date_aff	bperp_affdateList_affdim_refrows_ref	colms_refdim_affrows_aff	colms_aff
date_finalDate1Date2r   join_dimr   r   r   r   row_sumcol_sumts_join_datasetts_joinr   date1date2dis_refdis_ref_flattendis_affdis_aff_flattenr   ts_atrr!   r!   r"   concatenate_ts  sR   $




$

r&  c                 C   s   t |||||||\}}	}
}}}}}| j| }| j|	 }td t|\}}td t|\}}|d dkr@|d }|d }d}t|||
|||}|}|jd |d< |jd |d	< ||fS )
zconcantenate maskPS datar   r   	DATA_TYPEboolr   r   r   r   r   )r   r   r   r   r   r   r*   )r'   r   r   rq   rr   rs   rt   r=   r   r   rV   rW   r   r   r   r   r   r   msk_refmsk_ref_atrmsk_affmsk_aff_atrr   
msk_joinedmsk_atrr!   r!   r"   concatenate_mask  s    $

r/  c           .      C   s  | j d }| jd }|dkrtj| jd d d }|dkr,tj| jd d d }| jd }td tj	|ddd }tj	|d	dd }tj	|d
dd }tj	|ddd }tj	|ddd }td tj	|dd\}	}
tj	|d	d\}}tj	|d
d\}}tj	|dd\}}tj	|dd\}}td tj	|dd\}}tj	|d	d\}}tj	|d
d\}}tj	|dd\}}tj	|dd\}}|	
 }|
 }|
 }|
 } t|	}!t|}"td t|	|||}#t||||}$t|#|$\}%}&}'}(||%|&d |'|(d f })||%|&d |'|(d f }*||%|&d |'|(d f }+||%|&d |'|(d f },||%|&d |'|(d f }-|)|*|+|,|-|||| |!|"|#|$fS )zconcatenate geometry datar   r   z/inputs/geometryRadar.h5r   z+Read the geometry data for the whole regionlatituder   	longitudeincidenceAngleazimuthAngleheightz1Read the geometry data of the first given datasetz2Read the geometry data of the second given datasetz4Convert to X-Y coordinate based on the geometry info)	geo_file1	geo_file2ospathdirnamer   geo_fullr   r   r   r   r/   rQ   r\   ).r'   	data_geo1	data_geo2r;  lat_fulllon_fullinc_fullazi_fullhgt_fullrf   lat_atr1rg   lon_atr1inc1inc_atr1azi1azi_atr1hgt1hgt_atr1rh   lat_atr2ri   lon_atr2inc2inc_atr2azi2azi_atr2hgt2hgt_atr2rq   rr   rs   rt   r   r   r   r   r   r   r   r   
lat_joined
lon_joined
inc_joined
azi_joined
hgt_joinedr!   r!   r"   concatenate_geo  sR   


rX  c                 C   s   | j \}}t }t||d< t||d< d|d< t }| |d< |jd }|jd }	t|	du r8|d | d }
n|d | d |	 d }
tj||
|d	 d S )
Nr   r   r   	FILE_TYPEr   /.h5_datasetDictout_filemetadata)r*   r  r   r   
out_suffixr   r   write)r   r   datatyper'   rN   colmatr_velvel_data
output_diroutnamevel_filenamer!   r!   r"   	write_vel  s   


rj  c                 C   s   | d j dd  \}}|}t||d< t||d< d|d< |jd }|jd }	tj|jd }
t|	du r?|d | d }n|d | d	 |	 d }t	j
| ||d
 d S )Nr   r   r   r   rY  r   rZ  r[  r\  r]  )r*   r   r   ra  r8  r9  basenamer   r   r   rb  )ts_joined_datasetr%  r  rc  r'   rN   rd  atr_tsrg  rh  	file_namets_filenamer!   r!   r"   write_ts5  s   

rp  c                 C   s   | j \}}t }t||d< t||d< d|d< |d |d< |d dkr(| dk} t }| |d< |jd }|jd }	tj|jd }
t	|	du rQ|d | d	 }n|d | d
 |	 d	 }t
j|||d d S )Nr   r   maskrY  r'  r(  r   rZ  r[  r\  r]  )r*   r  r   r   ra  r8  r9  rk  r   r   r   rb  )r-  r.  rc  r'   rN   rd  atr_mskmsk_datarg  rh  rn  msk_filenamer!   r!   r"   
write_maskJ  s$   


ru  c                 C   s   | j \}}t }t||d< t||d< d|d< t }	| |	d< t }
||
d< t }||d< ||d< ||d	< | |d< ||d< |jd
 }|jd
 }t|d
u rPd}nd| }|d | d }tj|||d td d S )Nr   r   geometryrY  r1  r2  r4  r3  r5  r   geometryRadargeometryRadar_rZ  r[  r]  zFinish!)	r*   r  r   r   ra  r   r   rb  r   )rS  rT  rU  rV  rW  r'   rN   rd  atr_geolat_datalon_datageo_datarg  rh  geo_outnamegeo_filenamer!   r!   r"   	write_geof  s.   


r  c           
      C   s  | d d }| d d }| d d }| d d }|d d }|d d }|d d }|d d }	||krB||krB||krB||	krBdS ||krT||k rT||krT||	k rTdS ||k rf||krf||k rf||	krfdS ||krx||krx||krx||	krxdS ||kr||k r||kr||	krdS ||k r||k r||k r||	krdS ||k r||k r||kr||	k rdS ||k r||kr||kr||	krd	S d
S d
S d
S d
S )z)Find the reference data based on geo infor   r   r4   r3   r0   r7   r1   r2   r5   Nr!   )
r   r   row1_ulcol1_ulrow1_lrcol1_lrrow2_ulcol2_ulrow2_lrcol2_lrr!   r!   r"   find_the_reference  s2           r  c                 C   sv   |dks|dkrd}d}|}	| }
|}|}|}|}t d nd}d}| }	|}
|}|}|}|}t d |||	|
||||fS )Nr3   r0   r   r   z)The second dataset is the reference imagez(The first dataset is the reference image)r   )r   r   rq   rr   rs   rt   r=   r   r   rV   rW   r   r   r   r   r!   r!   r"   r     s(   
r   c                 C   s    | j d }t|}|d }|S )Nr   rY  )r   r   read_attribute)r'   r   data_atrrc  r!   r!   r"   determine_datatype  s   

r  c                 C   s  dd }dd }t d | jd }| jd }t d||| t d	||| t d
 | jd }|dkrDtj| jd d d }| jd }|dkrZtj| jd d d }| j	d }t d||| t d||| t d||| t d || || d S )Nc                 S   s   t j| rdS dS )NTrueFalse)r8  r9  exists)datasetr!   r!   r"   state_judge  s   z#list_and_check.<locals>.state_judgec                 S   sn   t j| ddd }t j| ddd }|d d |d d k r3|d d |d d k r3td|  d S td)Nr1  r   r   r2  r6   z{}: Correct lat and lon orderzkWrong lat/lon order! The lat should increase from north to south. The lon should increase from west to east)r   r   r   r   
ValueError)r|  r;   r<   r!   r!   r"   check_ordering  s   0z&list_and_check.<locals>.check_orderingz8Check and list the attribute dataset to be concatenated:r   r   z7---The first dataset is {} and the state of being is {}z8---The second dataset is {} and the state of being is {}z)Check and list the geometry dataset used:r   z	./inputs/r0  z@---The first geometry dataset is {} and the state of being is {}zA---The second geometry dataset is {} and the state of being is {}zJ---The geometry dataset of whole region is {} and the state of being is {}z)Check the latitude and longitude ordering)
r   r   r   r6  r8  r9  r:  
patch_filer7  r;  )r'   r  r  dataset1dataset2r<  r=  r;  r!   r!   r"   list_and_check  s,   




r  c                 C   s:  t | }|jrt| t  t|}td| td t|\}}}}}}}	}
}}}}}|jr8t|||||| td t	||}td|  |dkrct
|||	|
||||||
\}}t|||| d S |dkrt|||	|
||||||
\}}}t||||| d S |dkrt|||||	|
||\}}t|||| d S d S )NzData type is: zProcess the geometry infoz3Find the relative position between the two datasetszprocess %s datar   r   rq  )r(   listr  exitr  r   rX  	geo_writer  r  r   rj  r&  rp  r/  ru  )r&   r'   rc  rS  rT  rU  rV  rW  rq   rr   rs   rt   r   r   r   r   r=   r   r   r  r%  r  r-  r.  r!   r!   r"   main  s.   
"
 r  __main__rp   )6r8  r   numpyr+   r   copypandasr   rb   sklearn.neighborsr   	functoolsr   mintpymintpy.workflowmintpy.utilsr   r   r   r   utmintpy.objectsr   mimtpy.workflowr   r   r   r#   r(   r/   rD   rQ   r\   ra   ro   r   r   r   r   r   r   r&  r/  rX  rj  rp  ru  r  r  r   r  r  r  __name__r!   r!   r!   r"   <module>   sZ   	

%3S D:&
& 
