o
    n^f                     @   s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZ d dlm  mZ d dlZd dlmZ d dlmZ d dlZd dlZe Zd{ddZd	d
 Zdd Zdd Zdd Z dd Z!dd Z"dd Z#dd Z$dd Z%dd Z&dd Z'dd  Z(d!d" Z)d#d$ Z*d%d& Z+d'd( Z,d)d* Z-d+d, Z.d-d. Z/d|d0d1Z0d2d3 Z1d4d5 Z2d6d7 Z3d8d9 Z4d:d; Z5d<d= Z6d>d? Z7d@dA Z8dBdC Z9dDdE Z:dFdG Z;dHdI Z<dJdK Z=dLdM Z>dNdO Z?dPdQ Z@dRdS ZAdTdU ZBdVdW ZCdXdY ZDdZd[ ZEd\d] ZFd^d_ ZGd`da ZHdbdc ZIddde ZJdfdg ZKdhdi ZLd}dkdlZMdmdn ZNdodp ZOdqdr ZPdsdt ZQdudv ZRdwdx ZSdydz ZTdS )~    )print_functionN)Path)	natsorted)Template)PathFindc                 C   s   t jdd}t|}|dkrt|}|dkrt|}|dkr"t|}|dkr*t|}|dkr2t|}|dkr:t|}|d	krBt	|}|d
ksN|dksN|dkrRt
|}|j| d}t|}|S )zCommand line parser.zMinSAR scripts parser)descriptionZgenerate_download_commandZcreate_runfilesZ	dem_rsmasZexecute_runfilesZexport_amplitude_tifZemail_resultsZgenerate_chunk_template_filesZsmallbaseline_wrapperZingest_insarmapsZmiaplpy_wrapper)args)argparseArgumentParseradd_common_parseradd_generate_download_commandadd_create_runfilesadd_download_demadd_execute_runfilesadd_export_amplitudeadd_email_args&add_generate_chunk_template_files_argsadd_notification
parse_argscreate_or_update_template)iargsscriptparserinps r   F/home/exouser/operations/rsmas_insar/minsar/utils/process_utilities.pycmd_line_parse"   s*   r   c                 C   s   |  d}|jdddd |jdddd	d
 |jddddd |jddddd |jddddd |jddtddd |jdddd d | S )!NzGeneral options:custom_template_file?z&custom template with option settings.
)nargshelpz-vz	--versionversionz%(prog)s 0.1)actionr!   z--submitZsubmit_flag
store_truezsubmits jobdestr"   r    z
--walltime	wall_timezWALLTIME (HH:MM)z+walltime for submitting the script as a jobr%   metavarr    z--queuequeueZQUEUEzName of queue to submit job toz--reserveNodeZreserve_node   znumber of nodes to reserver%   typedefaultr    z--remoraZremoraz!use remora to get job informationadd_argument_groupadd_argumentint)r   Zcommonpr   r   r   r   ?   s   

r   c                 C   sX   |  d}|jdddddd |jddd	d
d |jddddd |jddddd | S )Nz"create run files and jobs options:z--tmpcopy_to_tmpr#   Tz?writes job files to run on /tmp rather than /scratch [default].r%   r"   r-   r    z--no-tmpstore_falsez5writes job files to run on /scratch rather than /tmp.r$   z
--jobfilesZ
write_jobsz*writes the jobs corresponding to run filesz--ignore_stackZignore_stackzVignores existing stack by temporay renaming /coreg_secondarys to /tmp_coreg_secondarysr/   r0   r   Z
run_parserr   r   r   r   O   s   



r   c                 C   sj   |  d}|jdddtdd |jddtd	d
 |jddtdd
 |jddtdd
 |jddtdd
 | S )NzDownload data options:z--delta_latZ	delta_latz0.0z?delta to add to latitude from boundingBox field, default is 0.0r%   r-   r,   r    z--seasonalStartDateZseasonalStartDatezseasonal start date to specify download dates within start and end dates, example: a seasonsal start date of January 1 would be added as --seasonalEndDate 0101r%   r,   r    z--seasonalEndDateZseasonalEndDatezseasonal end date to specify download dates within start and end dates, example: a seasonsal end date of December 31 would be added as --seasonalEndDate 1231z
--parallelZparallelz@determines whether a parallel download is required with a yes/noz--processes	processeszspecifies number of processes for the parallel download, if no value is provided then the number of processors from os.cpu_count() is used)r/   r0   floatstrr1   r   Zflag_parserr   r   r   r   \   s"   




r   c                 C   sJ   |  d}|jdddddd |jdd	ddd
d |jdddddd | S )NzDownload DEM flags:z--ssaraZ
flag_ssarar#   FzVrun ssara_federated_query w/ grd output file, set as default [option for dem_rsmas.py]r3   z--boundingBoxZflag_boundingBoxzYrun dem.py from isce using boundingBox as lat/long bounding box [option for dem_rsmas.py]z--ssara_kmlZflag_ssara_kmlz?get boundingBox from ssara_*.kml file [option for dem_rsmas.py]r5   r<   r   r   r   r   k   s(   
r   c                 C   s^   |  d}|jdddtdd |jdddtd	d |jd
dtddd |jddtddd | S )NzSteps of ISCE run files--startZ	start_runr   z(starting run file number (default = 1).
r7   --stopZend_runzstopping run file number.
--dostepstepSTEPz!run processing at the # step only)r%   r,   r(   r    z--numBurstsZ
num_burstsznumber of burstsz&number of bursts to calculate walltimer.   r6   r   r   r   r      s   
r   c                 C   s   |  d}|jdddtdd |jddd	td
dd |jdddtddd |jdddtdd |jdddtdd |jdddtddd |jdd d!td"d#d |jd$d%d&d'd( | S ))Nz2Options for exporting geo/ortho-rectified productsz-fz--fileZ
input_filez	Input SLCr8   z-lz--latZlat_filez
lat.rdr.mlz!latitude file in radar coordinater+   z-Lz--lonZlon_filez
lon.rdr.mlz"longitude file in radar coordinatez-y	--latSteplat_stepz(output pixel size in degree in latitude.z-xz	--lonStepZlon_stepz)output pixel size in degree in longitude.z-rz--resamplingMethodZresampling_methodZnearz.Resampling method (gdalwarp resamplin methods)-tz--typeZim_typeZorthoz
ortho, geoz--outDirout_dirZimage_productszoutput directory.r%   r-   r    )r/   r0   r;   r:   )r   Zproductsr   r   r   r      s,   
r   c                 C   sJ   |  d}|jdddddd |jddd	dd
d |jdddddd | S )Nz%Option for emailing insarmaps result.z--mintpyr#   Zemail_mintpy_flagFzEmail mintpy resultsr"   r%   r-   r    z	--miaplpyZemail_miaplpy_flagzEmail miaplpy resultsz--insarmapsZemail_insarmaps_flagzEmail insarmaps resultsr5   )r   Zemr   r   r   r      s   
r   c                 C   s   |  d}|jddddd |jddtd	d
d |jddtddd |jddddd |jddddd |jddddd |jddtddd | S )Nz-Options for generating  template file chunks.z--scriptZbash_scriptzminsarApp.bashzZbash script for command creation [minsarApp.bash|run_workflow_bash] (default: %(default)s)rF   rB   rC   g      ?z1chunk size in latitude [degrees] (default = 1.0)
r+   z--latMarginZ
lat_marging?z,margin to latStep [degrees] (default = 0.1)
r=   
start_steprA   z9start processing at the named step (default: %(default)s)r'   z--endend_stepz7end processing at the named step (default: %(default)s)r?   Zdo_step%run processing at the named step onlyz--wait	wait_time   z?wait time between job submission [secs] (default: %(default)s)
)r/   r0   r:   r1   )r   argr   r   r   r      s.   




r   c                 C   s"   |  d}|jdddddd | S )NzFlags for emailing results.z--emailr#   ZemailFzopt to email resultsrG   r5   )r   ZNOr   r   r   r      s
   
r   c              	   C   s   t  \}}| d|}|jddddd |jdddd	d |jd
dd|d d|d d |jddd|d d|d d |jddddd | S )Nz!steps processing (start/end/step)z-HZprint_templater#   z)print the default template file and exit.r$   z--removeProjectDirZremove_project_dirz'remove directory before download startsr=   rH   rA   r   z/start processing at the named step, default: {})r%   r(   r-   r    r>   rI   z-end processing at the named step, default: {}r?   r@   rJ   r'   )pathObjZprocess_rsmas_helpr/   r0   format)r   Z	STEP_LISTZ	STEP_HELPZprsr   r   r   add_process_rsmas   s$   


rQ   c                 C   s$   | D ]}t j|rt| qdS )z% Removes given existing directories. N)ospathisdirshutilrmtree)Zdirectories_to_delete	directoryr   r   r   remove_directories   s
   
rX   c                 C   s$   d}| rt jt j| d }|S )z2 Restores project name from custom template file. Nr   )rR   rS   splitextbasename)r   project_namer   r   r   get_project_name   s   
r\   c                 C   s>   t d| \}}d| v rd}nd| v rd}ntd|||fS )zT splits project name into location name, satellite and direction, and track number. zSenAT|SenDTZSenATZSenDTz.ERROR project name must contain SenDT or SenAT)resplit	Exception)r[   Zlocation_nameZ	sat_trackZsat_directionr   r   r   split_project_name   s   
r`   c                 C   s>   | sdt jv r|rt dd | } nt  } t j| } | S )z0 Sets the working directory under project name. 
SCRATCHDIR/)rR   environgetenvgetcwdrS   abspath)work_dirr[   r   r   r   get_work_directory  s   rh   c                 C   st   | }t d t d|j t|j|_t d|j td|j|_t d|j tj|js4tj	|jdd t
|}|S )z] Creates a default template file and/or updates it.
        returns the values in 'inps'
    z2
*************** Template Options ****************zCustom Template File: zProject Name: Nz
Work Dir: Texist_ok)printr   r\   r[   rh   rg   rR   rS   existsmakedirscreate_default_template)	inps_dictr   r   r   r   r     s   r   c              
   C   s  | }t j|j|_t j|jt j|j|_t|j}d|j	vr*t
d d|_n|j	d |_t|j}|D ]}||j	vrFtd|q8ttj}tdd}| D ]8}||D ]0\}}	|jdkrj|dps|jdkos|d}
|
r}|j	| q\|j	|t j|	di q\qU|j	|_t| |j	 D ]\}}|d	vr|j|t j|di qt|j|j t|j| t||_|j	t |j	 |! |_"|S )
zh
    :param temp_inps: input parsed arguments
    :return Updated template file added to temp_inps.
    Zacquisition_modez`WARNING: "acquisition_mode" is not given --> default: tops   (available options: tops, stripmap)ZtopszERROR: {0} is requiredzminsar_template_defaults.cfg)config_fileZstripmap')Nauto)#rR   rS   rf   r   joinrg   rZ   template_filer   optionsrk   prefixrO   Zrequired_template_optionsr_   rP   Zauto_templateget_config_defaultssectionsitems
startswithpopupdate
expandvarsstriptemplateZset_isce_defaultsrU   copyfileupdate_template_fileZgrab_cropboxZcropboxZcorrect_for_ssara_date_formatZgenerate_ssaraopt_stringssaraopt)Z	temp_inpsr   Zcustom_tempObjZrequired_template_keysZtemplate_keyZdefault_tempObjZconfig_templateZeach_sectionZeach_keyZeach_valstatuskeyvaluer   r   r   rn   3  sH   




 


rn   c              	   C   s   t | }d}|j D ]\}}||jvs|j| |kr"||j|< d}q|rftd d}|j D ]\}}|d| dd|d d	 }q0t| d
}|| W d   dS 1 s_w   Y  dS td|  dS )z
    updates final template file in project directory based on custom template file
    :param TEMP_FILE: file to be updated
    :param custom_templateObj: custom template having extra or new options
    :return: updated file text
    FTUpdating template filez&#####################################
{:<42}{:<15}= {}rq   
wNz$template file exists: {}, no updates)r   ru   ry   rk   rP   r~   open
writelines)	TEMP_FILEZcustom_templateObjtempObjZupdate_statusr   r   fileTextfiler   r   r   r   z  s(   
*
r   c              	   C   s   t d d}|j D ]\}}|d| dd|d d }qt| d}|| W d	   d	S 1 s:w   Y  d	S )
z
    writes template file in project directory based on custom template file
    :param TEMP_FILE: file to be writtem
    :param tempObj:  template to be written
    :return: written file text
    r   (#######################################
r   r   r   rq   r   r   N)rk   ru   ry   rP   r~   r   r   )r   r   r   r   r   r   r   r   r   write_template_file  s   *
r   c                 C   s   t | d}| }g d}g d}d}d}|D ]#}|}|t|k r:||| v r0||d |}n
|d }|t|k s!qt | d}|| dS )	z0
    adds lines with ##### to template file
    r)r   	topsStackmintpy)r   r   ZstripmapStackr   r   r   r*   r   N)r   	readlinesleninsertr   )r   flinesZdelimiter_listiZi_last_insertitemr   r   r   beautify_template_file  s&   


r   job_defaults.cfgc           	   	   C   s  t j}tj|| } tj| std| tj| dv r|g d}t	| d}|
 }W d   n1 s7w   Y  t }|D ]7}|dsN|drOqB| }t|dkry||d	  td	t|D ]}||d	 || ||d
   qgqB|S tjdd}t|_||  |S )z. Sets an optimized memory value for each job. z+job config file NOT found, it should be: {}r   )
c_walltime
s_walltimeZseconds_factorc_memorys_memoryZnum_threadsr2   Zio_loadr   N#z----   r   r*   =)Z
delimiters)rO   Z
defaultdirrR   rS   rs   isfile
ValueErrorrP   rZ   r   r   configparserZRawConfigParserrz   r^   r   Zadd_sectionrangesetZConfigParserr;   Zoptionxformread)	rp   Z
config_dirfieldsr   r   Zconfiglinerx   tr   r   r   rw     s0   
 
rw   c                 C   s&   t j| rt | t jrdS dS dS )z, Checks if the custom template file exists. runskipN)rR   rS   r   accessR_OK)r   r   r   r   run_or_skip  s
   r   c                 C   sL  ddl m} |}d}t| |\}}t|dkrdS t|}t|d }t|d }	t|dd}
t|
 t|	 |D ]}t	
|dd q9t|  t	j| d	 t	j|  }t	||d
  t|  |
|_t	jt	j||_t	j||_||}|j|d |j|d}t|d t|d t|t	jt	j| d t| dS )zUFind files that exited because walltime exceeed and run again with twice the walltimer   )
JOB_SUBMITzExited with exit code 140.N   )factorz.o*z.e*/stdout_Z
_pre_rerun)Z
batch_file)run_file)r   rg   )Zminsar.job_submissionr   *find_completed_jobs_matching_search_stringr   create_rerun_run_fileextract_walltime_from_job_fileextract_memory_from_job_filemultiply_walltimerk   rR   removereplacemove_out_job_files_to_stdoutrS   dirnamerZ   rename remove_last_job_running_productsr&   rg   rE   Zwrite_batch_jobsZsubmit_batch_jobs&remove_zero_size_or_length_error_filesraise_exception_if_job_exitedconcatenate_error_files)r   ro   r   r   search_stringfiles	job_files
rerun_filer&   memorynew_wall_timer   Z
stdout_dirZjob_objZjobsr   r   r   rerun_job_if_exit_code_140  s8   

r   c              	   C   s   | d d }zt | W n	 ty   Y nw t| }| D ]$}t|}t| t|d}|| W d   n1 s<w   Y  q|S )z+Write job file commands into rerun run filer   Z_rerunza+N)rR   r   OSErrorr   get_line_before_lastrk   r   write)r   r   r   r   Zcommand_liner   r   r   r   r   /  s   r   c                 C   sD   t | d}t|j}|| }W d   |S 1 sw   Y  |S )z
    extract attribute from an HDF5 file
    :param file: hdf file name
    :param attr: attribut to extracted
    :return Updated template file added to temp_inps.
    r   N)h5pyZFiledictattrs)r   Z	attributer   metadataZextracted_attributer   r   r   extract_attribute_from_hdf_fileH  s   


r   c                 C   s   t | F}| }|D ]6}d|v r'|dd }| }|  W  d   S d|v rA|dd }| }|  W  d   S qW d   dS 1 sMw   Y  dS )z3 Extracts the walltime from an LSF (BSUB) job file z#BSUB -Wz-Wr*   N
#SBATCH -trD   )r   r   r^   r~   )r   frr   r   Zwalltimer   r   r   r   Z  s    
"r   c                 C   sz   t | /}| }|D ]}d|v r*|dd }|dd }|  W  d   S qW d   dS 1 s6w   Y  dS )z1 Extracts the memory from an LSF (BSUB) job file zBSUB -R rusage[mem=zmem=r*   ]r   N)r   r   r^   )r   r   r   r   r   r   r   r   r   j  s   
"r   c                 C   s   t j| dd } | dd}|dd  r&d|ddd }|dd  r;d|ddd }|dd  rPd|ddd }|S )	z+ Extracts the step name from a stdout name z.or   run_ _rN   Nr*   )rR   rS   rZ   r^   r   isdigitrs   )Zjob_nameZ	step_namer   r   r   "extract_step_name_from_stdout_namev  s   r   c                 C   s0   z|  dd  dd }W |S    d}Y |S )z1 Extracts the config filename from a task string zconfigs/r*   r   r   r   r^   )Ztaskrp   r   r   r   $extract_config_file_from_task_string  s   r   c                 C   sN   |  dd }z	|  dd }W n   d}Y d}| r!|d }|| }|S )zZ Extracts the date string from config_file_name (last string if it does not contain date) r   rN   r   )r^   r   )Zconfig_file_nameZdate_or_string1Zdate_or_string0Zdate_stringr   r   r   )extract_date_string_from_config_file_name  s   r   c                 C   s<   t | }| }W d   n1 sw   Y  |d }|S )z$get the line before last from a fileNr   )r   r   )r   r   r   Zline_before_lastr   r   r   r     s
   

r   c           
   	   C   s   t  | d }g }t|}|D ](}t|}| }|D ]}||v r'|| qW d   n1 s2w   Y  qt|}g }|D ]}d|ddd d }	||	 q@||fS )zKreturns names of files that match seasrch strings (*.e files in run_files).*.o*Nr   r   rN   .job)globr   r   r   appendrs   r^   )
r   r   r   	file_listr   r   r   r   Zjob_file_listjob_filer   r   r   r     s&   

r   c              	   C   sz   t  | d }d}t|}|D ]+}t|}| }|D ]}||v r*td||qW d   n1 s5w   Y  qdS )ERemoves files with zero size or zero length (*.e files in run_files).r   zExited with exit codez ERROR: {0} exited; contains: {1}N)r   r   r   r   r_   rP   )r   r   r   r   r   r   r   r   r   r   r     s   
r   c              
   C   s8  t j|d | dd  d }t j|rt | t j| d | dd  d }t| d }t|}t	|dkst
|d;}|D ]0}|d |d	| d
  |d t
|}||  W d   n1 sqw   Y  qFW d   n1 sw   Y  t|  tt j|t j| dS )z
    Concatenate error files to one file (*.e files in run_files).
    :param directory: str
    :param out_name: str
    :return: None
    z/out_rb   rN   z.e*.e*r   zw+z#########################
z#### z 
N)rR   rS   rf   r^   r   r   r   r   r   r   r   r   r   r   touchrU   copy)r   rg   out_fileZout_nameerror_filesZoutfilefnameinfiler   r   r   r     s*   "
"


	r   c                 C   s:   t | d}t| W  d   S 1 sw   Y  dS )z(Calculate the number of lines in a file.r   N)r   r   r   )r   r   r   r   r   file_len	  s   
$r   c                 C   sd   t  | d t  | d  }t|}|D ]}tj|dkr$t| qt|dkr/t| qdS )r   r   r   r   N)r   r   rR   rS   getsizer   r   )r   r   r   r   r   r   r     s   
r   c                       t  | d }t|}|D ]=}g }t|d}| }|D ] t fdddD s.|  q|  t|d}|d| |  t	|
  qdS )	z=Removes launcher message from *.e files produced by launcher r   r   c                 3       | ]}| v V  qd S Nr   .0r   r   r   r   	<genexpr>*      z:remove_launcher_message_from_error_file.<locals>.<genexpr>)zusing /tmp/launcherzstarting job on r   r   Nr   r   r   r   anyr   closer   rs   r   r   r   r   r   	new_linesr   r   r   r   r   'remove_launcher_message_from_error_file      


r  c                    r   )	z9Removes warning message from *.e files when invoking ssh r   r   c                 3   r   r   r   r   r   r   r   r   A  r   z=remove_ssh_warning_message_from_error_file.<locals>.<genexpr>)zWarning: Permanently addedr   r   Nr   r   r   r   r   *remove_ssh_warning_message_from_error_file6  r  r  c                 C   s^   t  | d }t|}|D ]}g }t|d}| }|D ]}d|v r't| q|  qdS )z(Removes error files from timed out jobs r   r   zDUE TO TIME LIMIT ***N)r   r   r   r   rR   r   r   )r   r   r   r   r   r   r   r   r   r   remove_timeout_error_filesN  s   


r  c                 C   st   t  | d }t|}|D ]*}t|d}| }d|v r7|dd}tdd|}|  t|d}|| qdS )	zLRemoves lines with e.g. 'line:   398' from *.e files (filter_coherence step)r   r   z
line: r   z
\nline:\d+r   N)	r   r   r   r   r   r]   subr   r   )r   r   r   r   r   Ztmp_liner   r   r   r   *remove_line_counter_lines_from_error_filesb  s   


r  c                 C   s   t  | d }t|}|D ]H}t|d}| }d}d}d}d}g }	|D ]!}
|d7 }|r7|d7 }||kr6d}q$d|
v r@d}d}q$|	|
 q$t|d	}|	D ]}
||
 qMqd
S )z5 Remove 23 lines for dask distributed.worker - ERROR r   r   r      Fr*   zRdistributed.worker - ERROR - Failed to communicate with scheduler during heartbeatTr   N)r   r   r   r   r   r   )r   r   r   r   r   countZskip_numberZ
count_skipr   r   r   r   r   r   (remove_dask_error_lines_from_error_filesv  s4   

r
  c              	      s   t  | d }t|}dd}ddg |D ]]}t|d}| }fddt|D } fddt|D }t|t|krCtd	 ntt|t|D ]\}}	|||	d
 = qLt|d}
|
	| W d   n1 smw   Y  qdS )z9 Removes lines in smallbaseline*.o that occasionaly occurr   z"Traceback (most recent call last):z%distributed.comm.core.CommClosedErrorz"tornado.iostream.StreamClosedErrorr   c                    s   g | ]
\}} |v r|qS r   r   r   r   r   )start_phraser   r   
<listcomp>  s    z=remove_dask_error_lines_from_stdout_files.<locals>.<listcomp>c                    s   g | ]\}}t  |r|qS r   )search_for_end_phrasesr  )end_phrasesr   r   r    s    z+Mismatch in number of start and end phrasesr*   r   N)
r   r   r   r   	enumerater   rk   zipreversedr   )r   stdout_filesZ
end_phraser   r   r   Zstart_indicesZend_indicesstart_indexZ	end_indexr   r   )r  r  r   )remove_dask_error_lines_from_stdout_files  s&   

r  c                    s   t  fdd| D S )Nc                 3   r   r   r   )r   phraser   r   r   r     r   z)search_for_end_phrases.<locals>.<genexpr>)r   )r  r   r   r   r   r    s   r  c                 C   sZ   t  | d }t  | d }t  | d }|| | }t|dks+|D ]}t| q#d S )Nr   z*.jobr   r   )r   r   rR   r   )r   r   r   r   Z
list_filesr   r   r   r   r     s   r   c                 C   s   t  | d }t|dkrdS tj| }|d tj|  }tj|s-tj|dd nt	| tj|dd t|dkrJ|D ]}t
|| qAdS )z0move the stdout file into stdout_files directoryz*.or   Nr   Tri   r*   )r   r   rR   rS   r   rZ   rl   rm   rU   rV   move)r   r  Zdir_nameZ
out_folderr   r   r   r   r     s   
r   c                 C   s4   t  tj| tjd }tdd |}t|}|S )z- exports run files to a list: run_file_list. z/run_*c                 S   s   |  dd dvS )N.rN   )eoZjobr   )xr   r   r   <lambda>  s    zmake_run_list.<locals>.<lambda>)r   rR   rS   rs   rO   Zrundirfilterr   )rg   Zrun_listr   r   r   make_run_list  s   r  c                 C   sv   t j| d}g }t|d}| }W d   n1 sw   Y  |D ]}|| d |dd dd   q$|S )z reads from run_file_list. Zrun_files_listr   Nz/run_files/rb   rN   )rR   rS   rs   r   r   r   r^   )rg   ZrunfilesZrun_file_listr   new_fr   r   r   r   read_run_list  s   
&r   c                 C   s   t | }| }d}d}|d}|D ]}|ddkr"|d}q|d}|D ]}|ddv rD|D ]}|ddkrC|d}q5q*|jdkrMd	}	nd
}	|j|	d}
|
S )z% Reads attributes from isce xml file NZ	componentnameZmissionr   )Zburst1Zburst2Zburst3passdirectionZ
DESCENDINGZDescZAsc)Zmissionnamer"  )ETparsegetrootfindgettext)filenametreerootZvalue_node_spacecraftZvalue_node_passdirZnode_componentnodeZnode_burstspropertyr"  Zattrdictr   r   r   xmlread  s*   





r.  c                 C   s  ddl m} td}tjtjtdd ddlm	}m
} zt| }g }|D ]&}|dv r?|| dkr>|d	| g }q+|| d
urQ|d	| g || g }q+||}||\}	}
}}||
 j}|jd
u rmg d}n
dd |j D }d}|D ]g}| }|  | |_||_tj|jdd||_d
|_d
|_|j|_|j|_|j|_|jd
urdd |j D |_|tjd< z|   ||j!j" }W q{ t#y } z
t$| W Y d
}~q{d
}~ww W n   d}Y t$d| |S )zd calculates the number of bursts based on boundingBox and returns an adjusting factor for walltimes r   )	Sentinel1PATHZ
ISCE_STACKr   )cmdLineParse	get_dates)ZuseGPUZrmFilterTruez--N)r*   r      c                 S      g | ]}t |qS r   r1   )r   r   r   r   r   r  B      z(get_number_of_bursts.<locals>.<listcomp>Z	referencezIW{0}c                 S   r5  r   )r:   )r   r  r   r   r   r  R  r7  r*   znumber of bursts: {})%Zisceobj.Sensor.TOPS.Sentinel1r/  rR   rd   sysrS   r   rs   ZstackSentinelr1  r2  rO   Z"correct_for_isce_naming_conventionZ	safe_fileZ	swath_numr^   Z	configuresafeZswathNumberrg   rP   outputZ	orbitFileZauxFileZorbit_dirnameZorbitDirZaux_dirnameZauxDirZpolarizationZbboxZregionOfInterestrc   r$  productZnumberOfBurstsr_   rk   )ro   r/  Zsystem_pathZ	stack_cmdr2  ZtopsStack_templateZcommand_optionsr   r   ZdateListZreference_dateZsecondaryListZ	safe_dictr   ZswathsZnumber_of_burstsZswathobjr  r   r   r   get_number_of_bursts(  s`   







r=  SLURMc                 C   s  |dks%zt |d}W n
   t |d}Y tj|j|j|jd }nd}|dksLzt |d}W n
   t |d}Y tj|j|j|jd }	nd}	t|	t| t| |  }
|
|9 }
t	|
d\}}t	|d\}}|dv rzd||f }|S d	|||f }|S )
z
    scales default walltime by number of bursts
    scaled_walltime = c_walltime + (number_of_memory_units * s_walltime)*extra_seconds
    0z%H:%M:%Sz%H:%M)hoursminutessecondsr   <   )ZLSFz%d:%02dz%d:%02d:%02d)
timestrptimedatetime	timedeltatm_hourtm_mintm_sectotal_secondsr:   divmod)number_of_memory_unitsZwalltime_factorr   r   Zextra_secondsZ	schedulerZs_time_splitZs_time_secondsZc_time_splitZc_time_secondsZtime_secondsminZsechourZscaled_timer   r   r   scale_walltimec  sB   rP  c                 C   s2   |dkst |t | t |  }|S td}|S )zt
    scales default memory by number of bursts
    scaled_memory = c_memory + number_of_memory_units * s_memory
    allZMAX_MEMORY_PER_NODE)r:   rR   rd   )rM  r   r   Zscaled_memoryr   r   r   scale_memory  s
   
rR  c                 C   s0   dd |  dD }|d d |d  d }|S )Nc                 S   r5  r   r6  r   sr   r   r   r    r7  z!pause_seconds.<locals>.<listcomp>:r   rC  r*   r   )rK   Z
wait_partsZwait_secondsr   r   r   pause_seconds  s   rV  c           	      C   s   dd |  dD }|d }|d }z|d }W n   d}Y ||d  |d  }|| }t|d }t||d  d }t||d  |d  }d	|||}t|dkrad
||}|S d	|||}|S )z9 multiply walltime in HH:MM or HH:MM:SS format by factor c                 S   r5  r   r6  rS  r   r   r   r    r7  z%multiply_walltime.<locals>.<listcomp>rU  r   r*   r   rC    {:02d}:{:02d}:{:02d}{:02d}:{:02d})r^   mathfloorrP   r   )	r&   r   Zwall_time_partsr@  rA  rB  seconds_totalZseconds_newr   r   r   r   r     s$   r   c                 C   s   g }t | +}| }|D ]}d|v r$|dd }| }|||}|| qW d   n1 s4w   Y  |  t | d}|| W d   dS 1 sSw   Y  dS )z- replaces the walltime from a SLURM job file r   rD   r*   Nr   )r   r   r^   r~   r   r   r   r   )r   r   r   r   r   r   r&   r   r   r   r   replace_walltime_in_job_file  s$   

r]  c           
         sD  g }t  | d }t|}td|D ]}dt|d   fdd|D }q|D ]v}t|h}|| v rt|N}| }g }|D ]}	||	vrN||	 qC|	  t|d}|
| W d   n1 shw   Y  tj|d	krt| t|d
  W d   n1 sw   Y  W d   n1 sw   Y  q)dS )z removes dates from run_files z/run_*_*[0-9]r*   r   r   c                    s   g | ]} |vr|qS r   r   )r   r   stringr   r   r    s    z2run_remove_date_from_run_files.<locals>.<listcomp>r   Nr   r   )r   r   r   r;   zfillr   r   r   r   r   r   rR   rS   r   r   )
Zrun_files_dirdateZstart_run_fileZ	run_filesr   r   r   r   r   r   r   r^  r   run_remove_date_from_run_files  s:   



rb  c           
      C   s  | rd}| D ]J}| d}z|d  d\}}t|d t| }W n
   t|d }Y t|d }zt|d }W n   d}Y ||d  |d  }|| }qt|d }t||d  d }t||d  |d  }t|dkr~d	||}	|	S d
|||}	|	S d}	|	S )z* sum time in D-HH:MM or D-HH:MM:SS format r   rU  -   r*   r   rC  rW  rY  rX  z00:00:00)r^   r1   rZ  r[  r   rP   )
Ztime_str_listZseconds_sumr   Z
item_partsdaysr@  rA  rB  r\  Znew_time_strr   r   r   sum_time  s4   

rf  c                 C   sb   t  | d }t  | d }t  | d }t||}t||}t|dks/td| d S )Nz/worker*z	/worker*oz	/worker*er   zchmod -R 0755 {})r   npZ	setdiff1dr   rR   systemrP   )rW   ZworkersZworkers_outZworkers_errr   r   r   set_permission_dask_files(  s   ri  )NNr   )r   r>  )U
__future__r   rR   r8  r   r   r	   numpyrg  r   rZ  r]   pathlibr   Znatsortr   xml.etree.ElementTreeetreeElementTreer#  rU   Zminsar.objects.dataset_templater   Zminsar.objects.auto_defaultsr   rD  rF  rO   r   r   r   r   r   r   r   r   r   r   rQ   rX   r\   r`   rh   r   rn   r   r   r   rw   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r
  r  r  r   r   r  r   r.  r=  rP  rR  rV  r   r]  rb  rf  ri  r   r   r   r   <module>   s   	

G 
 "3&%$ 
;0
"'