B
    „éXaº­  ã               @   s‚  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZ d dlm  mZ d dlZd dlmZ d dlmZ d dlmZ d dlZd dlZeƒ Zdxdd	„Zd
d„ Zdd„ Z dd„ Z!dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd„ Z&dd„ Z'dd„ Z(dd„ Z)d d!„ Z*d"d#„ Z+d$d%„ Z,d&d'„ Z-d(d)„ Z.d*d+„ Z/d,d-„ Z0d.d/„ Z1d0d1„ Z2dyd3d4„Z3d5d6„ Z4d7d8„ Z5d9d:„ Z6d;d<„ Z7d=d>„ Z8d?d@„ Z9dAdB„ Z:dCdD„ Z;dEdF„ Z<dGdH„ Z=dIdJ„ Z>dKdL„ Z?dMdN„ Z@dOdP„ ZAdQdR„ ZBdSdT„ ZCdUdV„ ZDdWdX„ ZEdYdZ„ ZFd[d\„ ZGd]d^„ ZHd_d`„ ZIdadb„ ZJdcdd„ ZKdedf„ ZLdzdhdi„ZMdjdk„ ZNdldm„ ZOdndo„ ZPdpdq„ ZQdrds„ ZRdtdu„ ZSdvdw„ ZTdS ){é    )Úprint_functionN)ÚPath)Ú	natsorted)ÚTemplate)ÚPathFind)Ú	Sentinel1c             C   sÌ   t jdd}t|ƒ}|dkr$t|ƒ}|dkr4t|ƒ}|dkrDt|ƒ}|dkrTt|ƒ}|dkrdt|ƒ}|dkrtt|ƒ}|d	kr„t	|ƒ}|d
kr”t
|ƒ}|dks¬|dks¬|dkr´t|ƒ}|j| d}t|ƒ}|S )zCommand line parser.zMinSAR scripts parser)ÚdescriptionZdownload_rsmasZcreate_runfilesZ	dem_rsmasZexecute_runfilesZexport_amplitude_tifÚemail_resultsZupload_data_productsZgenerate_chunk_template_filesZsmallbaseline_wrapperÚingest_insarmapsZminopy_wrapper)Úargs)ÚargparseÚArgumentParserÚadd_common_parserÚadd_download_dataÚadd_create_runfilesÚadd_download_demÚadd_execute_runfilesÚadd_export_amplitudeÚadd_email_argsÚadd_upload_data_productsÚ&add_generate_chunk_template_files_argsÚadd_notificationÚ
parse_argsÚcreate_or_update_template)ÚiargsÚscriptÚparserÚinps© r   úE/home/centos/operations/rsmas_insar/minsar/utils/process_utilities.pyÚcmd_line_parse#   s.    r    c             C   s    |   d¡}|jdddd |jdddd	d
 |jddddd |jddddd |jddddd |jddtddd |jddd d!d"d# |jd$d%dd&d | S )'NzGeneral options:Úcustom_template_fileú?z&custom template with option settings.
)ÚnargsÚhelpz-vz	--versionÚversionz%(prog)s 0.1)Úactionr%   z--submitÚsubmit_flagÚ
store_truezsubmits job)Údestr&   r$   z
--walltimeÚ	wall_timezWALLTIME (HH:MM)z+walltime for submitting the script as a job)r)   Úmetavarr$   z--queueZqueueZQUEUEzName of queue to submit job toz--reserveNodeZreserve_nodeé   znumber of nodes to reserve)r)   ÚtypeÚdefaultr$   z--waitÚ	wait_timez00:00zWait time (hh:mm)zwait time to submit a job)r)   r.   r+   r$   z--remoraZremoraz!use remora to get job information)Úadd_argument_groupÚadd_argumentÚint)r   Zcommonpr   r   r   r   B   s    

r   c             C   sF   |   d¡}|jdddddd |jddd	d
d |jddddd | S )Nz"create run files and jobs options:z--tmpÚcopy_to_tmpr(   Fz?writes job files to run on /tmp rather than /scratch [default].)r)   r&   r.   r$   z--no_tmpZstore_falsez5writes job files to run on /scratch rather than /tmp.)r)   r&   r$   z
--jobfilesZ
write_jobsz*writes the jobs corresponding to run files)r0   r1   )r   Ú
run_parserr   r   r   r   R   s    


r   c             C   sj   |   d¡}|jdddtdd |jddtd	d
 |jddtdd
 |jddtdd
 |jddtdd
 | S )NzDownload data options:z--delta_latZ	delta_latz0.0z?delta to add to latitude from boundingBox field, default is 0.0)r)   r.   r-   r$   z--seasonalStartDateZseasonalStartDatezŸseasonal start date to specify download dates within start and end dates, example: a seasonsal start date of January 1 would be added as --seasonalEndDate 0101)r)   r-   r$   z--seasonalEndDateZseasonalEndDatezseasonal end date to specify download dates within start and end dates, example: a seasonsal end date of December 31 would be added as --seasonalEndDate 1231z
--parallelZparallelz@determines whether a parallel download is required with a yes/noz--processesZ	processeszŠspecifies number of processes for the parallel download, if no value is provided then the number of processors from os.cpu_count() is used)r0   r1   ÚfloatÚstrr2   )r   Úflag_parserr   r   r   r   ]   s    




r   c             C   s^   |   d¡}|jdddddd |jdd	ddd
d |jdddddd |jdddddd | S )Nzupload data products flagsz--mintpyProductsZmintpy_products_flagr(   Tz6uploads mintpy and minopy data products to data portal)r)   r&   r.   r$   z--minopyProductsZminopy_products_flagz6uploads minopy and mintpy data products to data portalz--allZmintpy_products_all_flagFzuploads full mintpy dirz--imageProductsZimage_products_flagz*uploads image data products to data portal)r0   r1   )r   r7   r   r   r   r   m   s,    
r   c             C   s6   |   d¡}|jdddddd |jdd	ddd
d | S )NzDownload DEM flags:z--ssaraZ
flag_ssarar(   FzVrun ssara_federated_query w/ grd output file, set as default [option for dem_rsmas.py])r)   r&   r.   r$   z--boundingBoxZflag_boundingBoxzYrun dem.py from isce using boundingBox as lat/long bounding box [option for dem_rsmas.py])r0   r1   )r   r7   r   r   r   r   ‡   s    
r   c             C   s^   |   d¡}|jdddtdd |jdddtd	d |jd
dtddd |jddtddd | S )NzSteps of ISCE run filesz--startZ	start_runr   z(starting run file number (default = 1).
)r)   r.   r-   r$   z--stopZend_runzstopping run file number.
z--dostepÚstepÚSTEPz!run processing at the # step only)r)   r-   r+   r$   z--numBurstsZ
num_burstsznumber of burstsz&number of bursts to calculate walltime)r0   r1   r2   )r   r4   r   r   r   r   —   s    
r   c             C   s´   |   d¡}|jdddtdd |jddd	td
dd |jdddtddd |jdddtdd |jdddtdd |jdddtddd |jdd d!td"d#d |jd$d%d&d'd( | S ))Nz2Options for exporting geo/ortho-rectified productsz-fz--fileZ
input_filez	Input SLC)r)   r-   r$   z-lz--latZlat_filez
lat.rdr.mlz!latitude file in radar coordinate)r)   r-   r.   r$   z-Lz--lonZlon_filez
lon.rdr.mlz"longitude file in radar coordinatez-yz	--latStepÚlat_stepz(output pixel size in degree in latitude.z-xz	--lonStepZlon_stepz)output pixel size in degree in longitude.z-rz--resamplingMethodZresampling_methodZnearz.Resampling method (gdalwarp resamplin methods)z-tz--typeZim_typeZorthoz
ortho, geoz--outDirÚout_dirZimage_productszoutput directory.)r)   r.   r$   )r0   r1   r6   r5   )r   Zproductsr   r   r   r   ¥   s     


r   c             C   sJ   |   d¡}|jdddddd |jddd	dd
d |jdddddd | S )Nz%Option for emailing insarmaps result.z--mintpyr(   Zemail_mintpy_flagFzEmail mintpy results)r&   r)   r.   r$   z--minopyZemail_minopy_flagzEmail minopy resultsz--insarmapsZemail_insarmaps_flagzEmail insarmaps results)r0   r1   )r   Zemr   r   r   r   ¹   s    
r   c             C   sJ   |   d¡}|jdddddd |jdd	td
dd |jddtddd | S )Nz-Options for generating  template file chunks.z
--downloadr(   Zdownload_flagFzdownload data for each chunk)r&   r)   r.   r$   z	--latStepr:   g      ð?z3chunk size in latitude [degrees] (default = 1.0).
.)r)   r-   r.   r$   z--latMarginZ
lat_margingš™™™™™¹?z-margin to latStep [degrees] (default = 0.1).
)r0   r1   r5   )r   Úargr   r   r   r   Ã   s    
r   c             C   s"   |   d¡}|jdddddd | S )NzFlags for emailing results.z--emailr(   ÚemailFzopt to email results)r&   r)   r.   r$   )r0   r1   )r   ZNOr   r   r   r   Î   s    
r   c          	   C   s–   t  ¡ \}}|  d|¡}|jddddd |jdddd	d |jd
dd|d d |d ¡d |jddd|d d |d ¡d |jddddd | S )Nz!steps processing (start/end/step)z-HZprint_templater(   z)print the default template file and exit.)r)   r&   r$   z--removeProjectDirZremove_project_dirz'remove directory before download startsz--startZ
start_stepr9   r   z/start processing at the named step, default: {})r)   r+   r.   r$   z--stopZend_stepéÿÿÿÿz-end processing at the named step, default: {}z--dostepr8   z%run processing at the named step only)r)   r+   r$   )ÚpathObjZprocess_rsmas_helpr0   r1   Úformat)r   Z	STEP_LISTZ	STEP_HELPZprsr   r   r   Úadd_process_rsmasÕ   s    


rA   c             C   s(   x"| D ]}t j |¡rt |¡ qW dS )z% Removes given existing directories. N)ÚosÚpathÚisdirÚshutilÚrmtree)Zdirectories_to_deleteÚ	directoryr   r   r   Úremove_directoriesê   s    
rH   c             C   s$   d}| r t j t j | ¡¡d }|S )z2 Restores project name from custom template file. Nr   )rB   rC   ÚsplitextÚbasename)r!   Úproject_namer   r   r   Úget_project_name÷   s
    rL   c             C   s>   t  d| ¡\}}d| krd}nd| kr,d}ntdƒ‚|||fS )zT splits project name into location name, satellite and direction, and track number. zSenAT|SenDTZSenATZSenDTz.ERROR project name must contain SenDT or SenAT)ÚreÚsplitÚ	Exception)rK   Zlocation_nameZ	sat_trackZsat_directionr   r   r   Úsplit_project_name  s    rP   c             C   s>   | s.dt jkr&|r&t  d¡d | } nt  ¡ } t j | ¡} | S )z0 Sets the working directory under project name. Z
SCRATCHDIRú/)rB   ÚenvironÚgetenvÚgetcwdrC   Úabspath)Úwork_dirrK   r   r   r   Úget_work_directory  s    rW   c             C   st   | }t dƒ t d|jƒ t|jƒ|_t d|jƒ td|jƒ|_t d|jƒ tj |j¡shtj	|jdd t
|ƒ}|S )z] Creates a default template file and/or updates it.
        returns the values in 'inps'
    z2
*************** Template Options ****************zCustom Template File: zProject Name: Nz
Work Dir: T)Úexist_ok)Úprintr!   rL   rK   rW   rV   rB   rC   ÚexistsÚmakedirsÚcreate_default_template)Ú	inps_dictr   r   r   r   r   %  s    r   c          
   C   sÚ  | }t j |j¡|_t j |jt j |j¡¡|_t|jƒ}d|j	krTt
dƒ d|_n|j	d |_t |j¡}x$|D ]}||j	krrtd |¡ƒ‚qrW ttjƒ}tdd}x‚| ¡ D ]v}xp| |¡D ]b\}}	|jdkrÜ| d¡pî|jdkoî| d¡}
|
r|j	 |¡ qÀ|j	 |t j |	 d¡¡i¡ qÀW q°W |j	|_t |¡ x@|j	 ¡ D ]2\}}|d	krH|j |t j | d¡¡i¡ qHW t j |j¡s t |j|j¡ nt|j|ƒ t |¡|_ |j	 t !|j	¡¡ | "¡ |_#|S )
zh
    :param temp_inps: input parsed arguments
    :return Updated template file added to temp_inps.
    Zacquisition_modez`WARNING: "acquisition_mode" is not given --> default: tops   (available options: tops, stripmap)ZtopszERROR: {0} is requiredzminsar_template_defaults.cfg)Úconfig_fileZstripmapú')NÚauto)$rB   rC   rU   r!   ÚjoinrV   rJ   Ztemplate_filer   ÚoptionsrY   Úprefixr?   Zrequired_template_optionsrO   r@   Zauto_templateÚget_config_defaultsÚsectionsÚitemsÚ
startswithÚpopÚupdateÚ
expandvarsÚstripÚtemplateZset_isce_defaultsrZ   rE   ÚcopyfileÚupdate_template_fileZgrab_cropboxZcropboxZcorrect_for_ssara_date_formatZgenerate_ssaraopt_stringÚssaraopt)Z	temp_inpsr   Zcustom_tempObjZrequired_template_keysZtemplate_keyZdefault_tempObjZconfig_templateZeach_sectionZeach_keyZeach_valÚstatusÚkeyÚvaluer   r   r   r\   B  sB    





&

$
r\   c          	   C   sÎ   t | ƒ}d}x<|j ¡ D ].\}}||jks8|j| |kr||j|< d}qW |r¼tdƒ d}x>|j ¡ D ]0\}}|d |¡ d d | d¡¡¡ d	 }qfW t| d
ƒ}| |¡ W dQ R X ntd | ¡ƒ dS )zë
    updates final template file in project directory based on custom template file
    :param TEMP_FILE: file to be updated
    :param custom_templateObj: custom template having extra or new options
    :return: updated file text
    FTzUpdating template filez&#####################################
z{:<38}z{:<15}z= {}r_   Ú
ÚwNz$template file exists: {}, no updates)r   rb   rf   rY   r@   rk   ÚopenÚ
writelines)Ú	TEMP_FILEZcustom_templateObjÚtempObjZupdate_statusrq   rr   ÚfileTextÚfiler   r   r   rn   ƒ  s    
,rn   c          	   C   sp   t dƒ d}x>|j ¡ D ]0\}}|d |¡ d d | d¡¡¡ d }qW t| dƒ}| |¡ W d	Q R X d	S )
zÅ
    writes template file in project directory based on custom template file
    :param TEMP_FILE: file to be writtem
    :param tempObj:  template to be written
    :return: written file text
    zUpdating template filez(#######################################
z{:<38}z{:<15}z= {}r_   rs   rt   N)rY   rb   rf   r@   rk   ru   rv   )rw   rx   ry   rq   rr   rz   r   r   r   Úwrite_template_file£  s    ,r{   c             C   s’   t | dƒ}| ¡ }dddg}ddddg}d}d}xH|D ]@}|}x6|t|ƒk rt||| krj| |d¡ |}P |d }q@W q6W t | d	ƒ}| |¡ d
S )z0
    adds lines with ##### to template file
    Úrro   Ú	topsStackZmintpyZstripmapStackr   z(#######################################
r,   rt   N)ru   Ú	readlinesÚlenÚinsertrv   )rw   ÚfÚlinesZdelimiter_listÚiZi_last_insertÚitemr   r   r   Úbeautify_template_file·  s"    




r…   újob_defaults.cfgc       	   	   C   s  t j}tj || ¡} tj | ¡s.td | ¡ƒ‚tj | ¡dkröddddddd	d
g}t	| dƒ}| 
¡ }W dQ R X t ¡ }x˜|D ]r}| d¡s~| d¡r˜q~| ¡ }t|ƒdkr~| |d ¡ x4tdt|ƒƒD ]"}| |d || ||d  ¡ qÊW q~W ntjdd}t|_| | ¡ |S )z. Sets an optimized memory value for each job. z+job config file NOT found, it should be: {})zjob_defaults.cfgÚ
c_walltimeÚ
s_walltimeÚextra_secondsÚc_memoryÚs_memoryZnum_threadsr3   Zio_loadr|   Nú#z----é   r   r,   ú=)Z
delimiters)r?   Z
defaultdirrB   rC   ra   ÚisfileÚ
ValueErrorr@   rJ   ru   r~   ÚconfigparserZRawConfigParserrg   rN   r   Zadd_sectionÚrangeÚsetZConfigParserr6   ZoptionxformÚread)	r^   Z
config_dirÚfieldsr   r‚   ZconfigÚlinere   Útr   r   r   rd   ×  s*    
(
rd   c             C   s(   t j | ¡r t  | t j¡r$dS ndS dS )z, Checks if the custom template file exists. ÚrunÚskipN)rB   rC   r   ÚaccessÚR_OK)r!   r   r   r   Úrun_or_skipù  s    rœ   c             C   sP  ddl m} |}d}t| |ƒ\}}t|ƒdkr2dS t|ƒ}t|d ƒ}t|d ƒ}	t|dd}
t|
ƒ t|	ƒ x|D ]}t	 
| dd¡¡ qtW t| ƒ t	j | ¡d	 t	j | ¡ }t	 ||d
 ¡ t| ƒ |
|_t	j t	j |¡¡|_t	j |¡|_||ƒ}|j|d |j|d}t|d t|d t|t	j t	j | ¡¡d t|ƒ dS )zUFind files that exited because walltime exceeed and run again with twice the walltimer   )Ú
JOB_SUBMITzExited with exit code 140.Né   )Úfactorz.o*z.e*z/stdout_Z
_pre_rerun)Z
batch_file)Úrun_file)r    rV   )Úminsar.job_submissionr   Ú*find_completed_jobs_matching_search_stringr   Úcreate_rerun_run_fileÚextract_walltime_from_job_fileÚextract_memory_from_job_fileÚmultiply_walltimerY   rB   ÚremoveÚreplaceÚmove_out_job_files_to_stdoutrC   ÚdirnamerJ   ÚrenameÚ remove_last_job_running_productsr*   rV   r;   Zwrite_batch_jobsZsubmit_batch_jobsÚ&remove_zero_size_or_length_error_filesÚraise_exception_if_job_exitedÚconcatenate_error_files)r    r]   r   r   Úsearch_stringÚfilesÚ	job_filesÚ
rerun_filer*   ÚmemoryÚnew_wall_timerz   Z
stdout_dirÚjob_objZjobsr   r   r   Úrerun_job_if_exit_code_140  s8    


r·   c          
   C   sz   | d d }yt  |¡ W n tk
r.   Y nX t| ƒ}x<| D ]4}t|ƒ}t|ƒ t|dƒ}| |¡ W dQ R X q>W |S )z+Write job file commands into rerun run filer   Z_rerunza+N)rB   r§   ÚOSErrorr   Úget_line_before_lastrY   ru   Úwrite)r²   r³   r±   rz   Zcommand_liner   r   r   r   r£   8  s    
r£   c          	   C   s.   t  | d¡}t|jƒ}|| }W dQ R X |S )z©
    extract attribute from an HDF5 file
    :param file: hdf file name
    :param attr: attribut to extracted
    :return Updated template file added to temp_inps.
    r|   N)Úh5pyZFileÚdictÚattrs)rz   Z	attributer   ZmetadataZextracted_attributer   r   r   Úextract_attribute_from_hdf_fileQ  s    
r¾   c          	   C   sr   t | ƒ`}| ¡ }xP|D ]H}d|kr>| d¡d }| ¡ }|S d|kr| d¡d }| ¡ }|S qW W dQ R X dS )z3 Extracts the walltime from an LSF (BSUB) job file z#BSUB -Wz-Wr,   z
#SBATCH -tz-tN)ru   r~   rN   rk   )rz   Úfrr‚   r–   Zwalltimer   r   r   r¤   c  s    

r¤   c          	   C   sV   t | ƒD}| ¡ }x4|D ],}d|kr| d¡d }| d¡d }|S qW W dQ R X dS )z1 Extracts the memory from an LSF (BSUB) job file zBSUB -R rusage[mem=zmem=r,   ú]r   N)ru   r~   rN   )rz   r¿   r‚   r–   r´   r   r   r   r¥   s  s    

r¥   c             C   s¤   t j | ¡ d¡d } |  dd¡}| d¡d  ¡ rLd | d¡dd… ¡}| d¡d  ¡ rvd | d¡dd… ¡}| d¡d  ¡ r d | d¡dd… ¡}|S )	z+ Extracts the step name from a stdout name z.or   Úrun_Ú Ú_r>   Nr,   )rB   rC   rJ   rN   r¨   Úisdigitra   )Újob_nameZ	step_namer   r   r   Ú"extract_step_name_from_stdout_name  s    rÆ   c             C   s2   y|   d¡d   d¡d }W n   d}Y nX |S )z1 Extracts the config filename from a task string zconfigs/r,   rs   r   rÂ   )rN   )Ztaskr^   r   r   r   Ú$extract_config_file_from_task_stringŽ  s
    
rÇ   c             C   sR   |   d¡d }y|   d¡d }W n   d}Y nX d}| ¡ rF|d }|| }|S )zZ Extracts the date string from config_file_name (last string if it does not contain date) rÃ   r>   éþÿÿÿrÂ   )rN   rÄ   )Zconfig_file_nameZdate_or_string1Zdate_or_string0Zdate_stringr   r   r   Ú)extract_date_string_from_config_file_nameš  s    
rÉ   c          	   C   s(   t | ƒ}| ¡ }W dQ R X |d }|S )z$get the line before last from a fileNrÈ   )ru   r~   )rz   r¿   r‚   Zline_before_lastr   r   r   r¹   ­  s
    
r¹   c       
   
   C   s¬   t   | d ¡}g }t|ƒ}xH|D ]@}t|ƒ.}| ¡ }x|D ]}||kr<| |¡ q<W W dQ R X q W t|ƒ}g }x2|D ]*}d | d¡dd… ¡d }	| |	¡ qvW ||fS )zKreturns names of files that match seasrch strings (*.e files in run_files).z*.o*NrÃ   r   r>   z.job)Úglobr   ru   r~   Úappendra   rN   )
r    r°   r±   Z	file_listrz   r¿   r‚   r–   Zjob_file_listÚjob_filer   r   r   r¢   ½  s    



r¢   c          
   C   sn   t   | d ¡}d}t|ƒ}xN|D ]F}t|ƒ4}| ¡ }x$|D ]}||kr<td ||¡ƒ‚q<W W dQ R X q W dS )zERemoves files with zero size or zero length (*.e files in run_files).z*.o*zExited with exit codez ERROR: {0} exited; contains: {1}N)rÊ   r   ru   r~   rO   r@   )r    r±   r°   rz   r¿   r‚   r–   r   r   r   r®   Ø  s    


r®   c             C   s  t j |¡d |  d¡d  d }t j |¡r8t  |¡ t j | ¡d |  d¡d  d }t | d ¡}t|ƒ}t	|ƒdksèt
|dƒ\}xT|D ]L}| d¡ | d	| d
 ¡ | d¡ t
|ƒ}| | ¡ ¡ W dQ R X qŽW W dQ R X t|ƒ ¡  t t j |¡t j |¡¡ dS )zŒ
    Concatenate error files to one file (*.e files in run_files).
    :param directory: str
    :param out_name: str
    :return: None
    z/out_rQ   r>   z.ez*.e*r   zw+z#########################
z#### z 
N)rB   rC   rU   rN   r   r§   rª   rÊ   r   r   ru   rº   r”   r   ÚtouchrE   Úcopy)r    rV   Úout_fileZout_nameÚerror_filesZoutfileÚfnameÚinfiler   r   r   r¯   ì  s"    "
"



&r¯   c          	   C   s"   t | dƒ}t| ¡ ƒS Q R X dS )z(Calculate the number of lines in a file.r|   N)ru   r   r~   )rÑ   rz   r   r   r   Úfile_len  s    rÓ   c             C   sh   t   | d ¡t   | d ¡ }t|ƒ}x>|D ]6}tj |¡dkrJt |¡ q*t|ƒdkr*t |¡ q*W dS )zERemoves files with zero size or zero length (*.e files in run_files).z*.e*z*.o*r   N)rÊ   r   rB   rC   Úgetsizer§   rÓ   )r    rÐ   r„   r   r   r   r­     s    
r­   c                s¢   t   | d ¡}t|ƒ}x†|D ]~}g }t|dƒ}| ¡ }x,|D ]$‰ t‡ fdd„dD ƒƒs<| ˆ ¡ q<W | ¡  t|dƒ}| d |¡¡ | ¡  t	|ƒ 
¡  qW dS )	z=Removes launcher message from *.e files produced by launcher z*.e*r|   c             3   s   | ]}|ˆ kV  qd S )Nr   )Ú.0r™   )r–   r   r   ú	<genexpr>3  s    z:remove_launcher_message_from_error_file.<locals>.<genexpr>)zusing /tmp/launcherzstarting job on rt   rÂ   N)rÊ   r   ru   r~   ÚanyrË   Úcloserº   ra   r   rÍ   )r    rÐ   r„   Ú	new_linesr   r‚   r   )r–   r   Ú'remove_launcher_message_from_error_file(  s    



rÚ   c                s¢   t   | d ¡}t|ƒ}x†|D ]~}g }t|dƒ}| ¡ }x,|D ]$‰ t‡ fdd„dD ƒƒs<| ˆ ¡ q<W | ¡  t|dƒ}| d |¡¡ | ¡  t	|ƒ 
¡  qW dS )	z9Removes warning message from *.e files when invoking ssh z*.e*r|   c             3   s   | ]}|ˆ kV  qd S )Nr   )rÕ   r™   )r–   r   r   rÖ   J  s    z=remove_ssh_warning_message_from_error_file.<locals>.<genexpr>)zWarning: Permanently addedrt   rÂ   N)rÊ   r   ru   r~   r×   rË   rØ   rº   ra   r   rÍ   )r    rÐ   r„   rÙ   r   r‚   r   )r–   r   Ú*remove_ssh_warning_message_from_error_file?  s    



rÛ   c             C   sf   t   | d ¡}t|ƒ}xJ|D ]B}g }t|dƒ}| ¡ }x|D ]}d|kr<t |¡ q<W | ¡  qW dS )z(Removes error files from timed out jobs z*.e*r|   zDUE TO TIME LIMIT ***N)rÊ   r   ru   r~   rB   r§   rØ   )r    rÐ   r„   rÙ   r   r‚   r–   r   r   r   Úremove_timeout_error_filesW  s    


rÜ   c             C   sx   t   | d ¡}t|ƒ}x\|D ]T}t|dƒ}| ¡ }d|kr| dd¡}t dd|¡}| ¡  t|dƒ}| |¡ qW dS )	zLRemoves lines with e.g. 'line:   398' from *.e files (filter_coherence step)z*.e*r|   z
line:ú rÂ   z
\nline:\d+rt   N)	rÊ   r   ru   r”   r¨   rM   ÚsubrØ   rº   )r    rÐ   r„   r   r‚   Ztmp_linerÙ   r   r   r   Ú*remove_line_counter_lines_from_error_filesk  s    


rß   c             C   s^   t   | d ¡}t   | d ¡}t   | d ¡}|| | }t|ƒdksZx|D ]}t |¡ qHW d S )Nz*.e*z*.jobz*.o*r   )rÊ   r   rB   r§   )r    rÐ   r²   rÏ   Z
list_filesr„   r   r   r   r¬   €  s    
r¬   c             C   sœ   t   | d ¡}t|ƒdkrdS tj | ¡}|d tj | ¡ }tj |¡sZtj|dd nt 	|¡ tj|dd t|ƒdkr˜x|D ]}t 
||¡ q„W dS )z0move the stdout file into stdout_files directoryz*.or   Nz/stdout_T)rX   r,   )rÊ   r   rB   rC   rª   rJ   rZ   r[   rE   rF   Úmove)r    Zstdout_filesZdir_nameZ
out_folderr„   r   r   r   r©   Ž  s    

r©   c             C   s4   t   tj | tj¡d ¡}tdd„ |ƒ}t|ƒ}|S )z- exports run files to a list: run_file_list. z/run_*c             S   s   |   d¡d dkS )NÚ.r>   )ÚeÚoZjob)rN   )Úxr   r   r   Ú<lambda>¬  ó    zmake_run_list.<locals>.<lambda>)rÊ   rB   rC   ra   r?   ZrundirÚfilterr   )rV   Zrun_listr   r   r   Úmake_run_list¨  s    rè   c          	   C   sf   t j | d¡}g }t|dƒ}| ¡ }W dQ R X x0|D ](}| | d | d¡d dd…  ¡ q6W |S )z reads from run_file_list. Zrun_files_listr|   Nz/run_files/rQ   r>   )rB   rC   ra   ru   r~   rË   rN   )rV   ZrunfilesZrun_file_listr   Znew_fr–   r   r   r   Úread_run_list´  s    
(ré   c             C   sº   t  | ¡}| ¡ }d}d}| d¡}x$|D ]}| d¡dkr*| d¡}q*W | d¡}x@|D ]8}| d¡dkrZx$|D ]}| d¡dkrr| d¡}qrW qZW |jdkr¦d	}	nd
}	|j|	dœ}
|
S )z% Reads attributes from isce xml file NZ	componentÚnameZmissionrr   )Zburst1Zburst2Zburst3ÚpassdirectionZ
DESCENDINGZDescZAsc)Zmissionnamerë   )ÚETÚparseZgetrootÚfindÚgetÚtext)ÚfilenameZtreeÚrootZvalue_node_spacecraftZvalue_node_passdirZnode_componentÚnodeZnode_burstsÚpropertyrë   Zattrdictr   r   r   ÚxmlreadÄ  s&    






rõ   c             C   sð  t  d¡}tj t j t  d¡d¡¡ ddlm}m} y”t	 
| ¡}g }xT|D ]L}|dkrv|| dkrš|d| g }qN|| d	k	rN|d| g || g }qNW ||ƒ}||ƒ\}}	}
}||	 j}|jd	krÖd
ddg}ndd„ |j ¡ D ƒ}d}xÚ|D ]Ò}tƒ }| ¡  | ¡ |_||_t j |jdd |¡¡|_d	|_d	|_|j|_|j|_|j|_|jd	k	rvdd„ |j ¡ D ƒ|_|t jd< y| ¡  ||j j! }W qô t"k
rÄ } zt#|ƒ W d	d	}~X Y qôX qôW W n   d
}Y nX t#d |¡ƒ |S )zd calculates the number of bursts based on boundingBox and returns an adjusting factor for walltimes ÚPATHZ
ISCE_STACKr}   r   )ÚcmdLineParseÚ	get_dates)ZuseGPUZrmFilterÚTruez--Nr,   rž   é   c             S   s   g | ]}t |ƒ‘qS r   )r2   )rÕ   rƒ   r   r   r   ú
<listcomp>ý  s    z(get_number_of_bursts.<locals>.<listcomp>Z	referencezIW{0}c             S   s   g | ]}t |ƒ‘qS r   )r5   )rÕ   rä   r   r   r   rû     s    znumber of bursts: {})$rB   rS   ÚsysrC   rË   ra   ZstackSentinelr÷   rø   r?   Z"correct_for_isce_naming_conventionZ	safe_fileZ	swath_numrN   r   Z	configureZsafeZswathNumberrV   r@   ÚoutputZ	orbitFileZauxFileZorbit_dirnameZorbitDirZaux_dirnameZauxDirZpolarizationZbboxZregionOfInterestrR   rí   ÚproductZnumberOfBurstsrO   rY   )r]   Zsystem_pathZ	stack_cmdrø   ZtopsStack_templateZcommand_optionsr„   r   ZdateListZreference_dateZsecondaryListZ	safe_dictrª   ZswathsÚnumber_of_burstsZswathÚobjrâ   r   r   r   Úget_number_of_burstsä  sT    







"
r  ÚSLURMc             C   s  |dksNyt  |d¡}W n   t  |d¡}Y nX tj|j|j|jd ¡ }nd}|dks yt  |d¡}W n   t  |d¡}Y nX tj|j|j|jd ¡ }	nd}	t|	ƒt| ƒt|ƒ  | }
|
|9 }
t	|
dƒ\}}t	|dƒ\}}|dkrúd||f }nd	|||f }|S )
zv
    scales default walltime by number of bursts
    scaled_walltime = c_walltime + number_of_bursts * s_walltime
    Ú0z%H:%M:%Sz%H:%M)ÚhoursÚminutesÚsecondsr   é<   )ZLSFz%d:%02dz%d:%02d:%02d)
ÚtimeÚstrptimeÚdatetimeÚ	timedeltaÚtm_hourÚtm_minÚtm_secÚtotal_secondsr5   Údivmod)rÿ   Zwalltime_factorr‡   rˆ   r‰   Z	schedulerZs_time_splitZs_time_secondsZc_time_splitZc_time_secondsZtime_secondsÚminZsecÚhourZscaled_timer   r   r   Úscale_walltime  s4    r  c             C   s0   |dks"t |ƒt | ƒt |ƒ  }n
t d¡}|S )zn
    scales default memory by number of bursts
    scaled_memory = c_memory + number_of_bursts * s_memory
    ÚallZMAX_MEMORY_PER_NODE)r5   rB   rS   )rÿ   rŠ   r‹   Zscaled_memoryr   r   r   Úscale_memoryN  s    
r  c             C   s0   dd„ |   d¡D ƒ}|d d |d  d }|S )Nc             S   s   g | ]}t |ƒ‘qS r   )r2   )rÕ   Úsr   r   r   rû   _  s    z!pause_seconds.<locals>.<listcomp>ú:r   r  r,   )rN   )r/   Z
wait_partsZwait_secondsr   r   r   Úpause_seconds^  s    r  c       	      C   sÖ   dd„ |   d¡D ƒ}|d }|d }y|d }W n   d}Y nX ||d  |d  }|| }t |d ¡}t ||d  d ¡}t ||d  |d  ¡}d	 |||¡}t|ƒdkrÄd
 ||¡}nd	 |||¡}|S )z9 multiply walltime in HH:MM or HH:MM:SS format by factor c             S   s   g | ]}t |ƒ‘qS r   )r2   )rÕ   r  r   r   r   rû   k  s    z%multiply_walltime.<locals>.<listcomp>r  r   r,   rž   r  i  z{:02d}:{:02d}:{:02d}z{:02d}:{:02d})rN   ÚmathÚfloorr@   r   )	r*   rŸ   Zwall_time_partsr  r  r  Úseconds_totalZseconds_newrµ   r   r   r   r¦   h  s"    
r¦   c          	   C   sŽ   g }t | ƒP}| ¡ }x@|D ]8}d|krJ| d¡d }| ¡ }| ||¡}| |¡ qW W dQ R X | ¡  t | dƒ}| |¡ W dQ R X dS )z- replaces the walltime from a SLURM job file z
#SBATCH -tz-tr,   Nrt   )ru   r~   rN   rk   r¨   rË   rØ   rv   )rz   rµ   rÙ   r¿   r‚   r–   r*   rÌ   r   r   r   Úreplace_walltime_in_job_file†  s    

r  c       
         s  g }t   | d ¡}t|ƒ}x6td|ƒD ](}dt|ƒ d¡ ‰ ‡ fdd„|D ƒ}q&W x¾|D ]¶}t|ƒ¤}|| ¡ krt|ƒ‚}| ¡ }g }x|D ]}	||	kr| |	¡ qW | 	¡  t|dƒ}| 
|¡ W dQ R X tj |¡d	krút |¡ t |d
 ¡ W dQ R X W dQ R X qXW dS )z removes dates from run_files z/run_*_*[0-9]r,   rÁ   rž   c                s   g | ]}ˆ |kr|‘qS r   r   )rÕ   r„   )Ústringr   r   rû   £  s    z2run_remove_date_from_run_files.<locals>.<listcomp>rt   Nr   z.job)rÊ   r   r’   r6   Úzfillru   r”   r~   rË   rØ   rv   rB   rC   rÔ   r§   )
Zrun_files_dirÚdateZstart_run_fileZ	run_filesrƒ   r    r   r‚   rÙ   r–   r   )r  r   Úrun_remove_date_from_run_filesš  s,    




&r   c       
      C   s"  | rd}x¤| D ]œ}|  d¡}y*|d   d¡\}}t|ƒd t|ƒ }W n   t|d ƒ}Y nX t|d ƒ}yt|d ƒ}W n   d}Y nX ||d  |d  }|| }qW t |d ¡}t ||d  d ¡}t ||d  |d  ¡}t|ƒdkr
d	 ||¡}	nd
 |||¡}	nd}	|	S )z* sum time in D-HH:MM or D-HH:MM:SS format r   r  ú-é   r,   rž   r  i  z{:02d}:{:02d}z{:02d}:{:02d}:{:02d}z00:00:00)rN   r2   r  r  r   r@   )
Ztime_str_listZseconds_sumr„   Z
item_partsÚdaysr  r  r  r  Znew_time_strr   r   r   Úsum_time¼  s0    


r$  c             C   sb   t   | d ¡}t   | d ¡}t   | d ¡}t ||¡}t ||¡}t|ƒdks^t d |¡¡ d S )Nz/worker*z	/worker*oz	/worker*er   zchmod -R 0755 {})rÊ   ÚnpZ	setdiff1dr   rB   Úsystemr@   )rG   ZworkersZworkers_outZworkers_errr   r   r   Úset_permission_dask_filesã  s    r'  )NN)r†   )r   r  )UÚ
__future__r   rB   rü   rÊ   r‘   r   Znumpyr%  r»   r  rM   Zpathlibr   Znatsortr   Zxml.etree.ElementTreeZetreeZElementTreerì   rE   Zminsar.objects.dataset_templater   Zminsar.objects.auto_defaultsr   Zisceobj.Sensor.TOPS.Sentinel1r   r  r
  r?   r    r   r   r   r   r   r   r   r   r   r   rA   rH   rL   rP   rW   r   r\   rn   r{   r…   rd   rœ   r·   r£   r¾   r¤   r¥   rÆ   rÇ   rÉ   r¹   r¢   r®   r¯   rÓ   r­   rÚ   rÛ   rÜ   rß   r¬   r©   rè   ré   rõ   r  r  r  r  r¦   r  r   r$  r'  r   r   r   r   Ú<module>
   s’   

A  
"3& :
0
"'