U
    7ãb ã                   @   s,  d Z ddlZddlZddlZddlZddlZddlZddlZddlZ	ddl
Z
ddlZddlmZ ddlmZmZ ddlZddlm  mZ ddlmZ ddlZejded dd	„ Zd
d„ ZG dd„ dƒZdd„ Zdd„ Zdd„ Ze dkr(eej!dd… ƒZ"ee"ƒZ#e#j$e"j%d e"j&dkr(e# '¡ Z(dS )a‡  
Functions related to batch job submission.
Should be run with a file containing jobs to submit as a batch.
Optional parameters for job submission are, --memory, --walltime, and --queuename.
Generates job scripts, runs them, and waits for output files to be written before exiting.


This script has functions to support submitting two different job types: a script as a job or a batch file consisting of
multiple parallel tasks. submitting a script as a job is done calling the function: submit_script
However submitting a batch file (calling submit_batch_jobs) can be done in 5 different ways with or without launcher.
Two environmental variables have to be set: JOB_SUBMISSION_SCHEME and QUEUENAME  (set in '~/accounts/platforms_defaults.bash')
QUEUENAME has defaults based on platforms. comment/uncomment or introduce a new one
JOB_SUBMISSION_SCHEME: it can have one of these options:

singleTask                     ---> submit each task of a batch file separately in a job
multiTask_singleNode           ---> distribute tasks of a batch file into jobs with one node
multiTask_multiNode            ---> submit tasks of a batch file in one job with required number of nodes
launcher_multiTask_singleNode  ---> distribute tasks of a batch file into jobs with one node, submit with launcher
launcher_multiTask_multiNode   ---> submit tasks of a batch file in one job with required number of nodes using launcher

é    N)Úmessage_rsmas)Úqueue_config_fileÚsupported_platforms)ÚdatetimeÚignore)Úcategoryc                  C   s  t jt jd} |  dd¡}|jdtdd |jddtd	d
d |jddddd |jddddd |jddddd |jdddddd |jdd td!d"d |jd#d$td%d&d'd( |jd)d*td+d,d-d. |jd/d0d1d2d3 |jd4d5d1d6d3 |jd7d8d1d9d3 | S ):a  
    Creates an argument parser for parsing parameters for batch job submission.
    Required parameter: file to batch create
    Optional parameters: template file, memory, walltime, and queue name
    :return: ArgumentParser object for parsing command line batch job submission
    )Úformatter_classz
Input FilezFile/Dataset to displayÚfilezThe file to batch create)ÚtypeÚhelpz
--templateÚcustom_template_fileztemplate filez&custom template with option settings.
)Údestr
   Úmetavarr   z--memoryÚmemoryzMEMORY (KB)z4Amount of memory to allocate, specified in kilobytes)r   r   r   z
--walltimeÚ	wall_timezWALLTIME (HH:MM)z+Amount of wall time to use, in HH:MM formatz--queueÚqueueÚQUEUEzName of queue to submit job toz--outdirÚout_dirÚ	run_filesZOUTDIRzoutput directory for run files)r   Údefaultr   r   z--numBurstsÚ
num_burstsznumber of burstsz&number of bursts to calculate walltimez	--numDataÚnum_dataé   znumber of dataz;number of data (interferogram or slc) to calculate walltime)r   r
   r   r   r   z--distributeÚ
distributeú+NzFiles to be copied to /tmp)r   r
   Únargsr   r   z--writeonlyÚ	writeonlyÚ
store_truez"Write job files without submitting)r   Úactionr   z--remoraÚremoraz!use remora to get job informationz--tmpÚcopy_to_tmpz!write to run_files_tmp/ directory)ÚargparseÚArgumentParserÚRawTextHelpFormatterÚadd_argument_groupÚadd_argumentÚstrÚint)ÚparserÚgroup© r*   ú</home/centos/operations/rsmas_insar/minsar/job_submission.pyÚcreate_argument_parser.   s<    
 ÿ
ÿ
ÿÿÿÿÿr,   c                 C   s¦   t ƒ }| | ¡}zt d¡}W n   t ¡ }Y nX tj |j¡|_tj ||j 	tj 
|¡¡d  d¡d ¡|_|jr„|jd |_d|jkr¢tj |j|j¡|_|S )z·
    Parses command line arguments into namespace.
    :param args: Arguments to parse
    :return: Namespace with submission parameters (from command line arguments or defaults)
    ZSCRATCHr   ú/Ú_tmpr   )r,   Ú
parse_argsÚosÚgetenvÚgetcwdÚpathÚabspathr	   ÚjoinÚrsplitÚbasenameÚsplitÚwork_dirr    r   )Úargsr(   Z
job_paramsZscratch_dirr*   r*   r+   Úparse_argumentsO   s    
 ÿ
r;   c                   @   sŽ   e Zd ZdZdd„ Zd dd„Zd!dd	„Zd"d
d„Zdd„ Zd#dd„Z	d$dd„Z
d%dd„Zd&dd„Zd'dd„Zd(dd„Zd)dd„Zd*dd„ZdS )+Ú
JOB_SUBMITz<
        A class representing the job submission object
    c              
   C   s¼  |j  ¡ D ]}t| ||j | ƒ q
d|krR|jdkrRtj t d¡d¡| _d| _ntj t d¡d¡| _d| _t	|ƒ\	| _
| _| _| _| _| _| _| _| _d| _d|ks®|js´d | _d	|ksÂ|jsÈd | _d
|ksÖ|jsÜd | _d|ksê|jsò| j| _d|krd| _d|krd | _d|kr"d | _|j| _d | _d | _d | _d|ksN|jsTd| _d| _ g | _!z0t" "| j#d ¡d }tj $|¡|j%|jd < W n    t&d '| j#d ¡ƒ Y nX || _(d S )NÚprefixÚstripmapÚ
ISCE_STACKÚstripmapStackÚ	topsStackÚtopsr   r   r   r   r   r   Ú.r   r    Úreserve_nodeTz/DEM/*.wgs84r   zStack.demDirzDEM does not exist in {}z/DEM))Ú__dict__ÚkeysÚsetattrr=   r0   r3   r5   r1   Ú
stack_pathÚset_job_queue_valuesÚsubmission_schemeÚplatform_nameÚ	schedulerÚ
queue_nameÚnumber_of_cores_per_nodeÚnumber_of_threads_per_coreÚmax_jobs_per_workflowÚmax_memory_per_nodeÚwall_time_factorÚ!number_of_parallel_tasks_per_noder   r   r   r   r   r   r    r   Údefault_memoryÚdefault_wall_timeÚdefault_num_threadsrD   Úemail_notifÚ	job_filesÚglobr9   ÚdirnameÚtemplateÚprintÚformatÚinps)Úselfr^   ÚkZdem_filer*   r*   r+   Ú__init__n   sV    þ   


zJOB_SUBMIT.__init__NÚFalsec                 C   s¨   t j | j¡s4t j | j¡r(t  | j¡ t  | j¡ |dk	rB|| _d dd„ |dd… D ƒ¡}| j	|dd g | _
| j|||| j| jd |d	kr¤| j| j
| jd
 dS )aŸ  
        Submits a single script as a job. (compare to submit_batch_jobs for several tasks given in run_file)
        :param job_name: Name of job.
        :param job_file_name: Name of job file.
        :param argv: Command line arguments for running job.
        :param email_notif: If email notifications should be on or not. Defaults to true.
        :return job number of the script that was submitted
        Nú c                 s   s   | ]}|d kr|V  qdS )z--submitNr*   )Ú.0Úflagr*   r*   r+   Ú	<genexpr>±   s      z+JOB_SUBMIT.submit_script.<locals>.<genexpr>r   Úscript©Újob_type©r9   Únumber_of_nodesrb   ©r9   )r0   r3   Úisdirr9   ÚisfileÚremoveÚmakedirsrW   r5   Úget_memory_walltimerX   Úwrite_single_job_filerD   Úsubmit_and_check_job_status)r_   Újob_nameÚjob_file_nameÚargvrW   Z	writeOnlyÚcommand_liner*   r*   r+   Úsubmit_scriptŸ   s    	ÿzJOB_SUBMIT.submit_scriptc              	   C   sÆ  |dkr| j }|dk	r|| _g | _| jtkrÂ| jdkr>d| _| j|dd dj| j	|| j
| jd}| jrr|d7 }|dk	r´|d	7 }t|tƒr¦|D ]}|d
 |¡7 }qn|d
 |¡7 }t | j|¡ t|dƒ}| ¡ }t|ƒ}	W 5 Q R X t t |	t| jƒ | j| j  ¡¡}
|dk	r,| j| | _|
|7 }
d| jkrJ| j||
|d nxd| jkr¢|d }tj  |¡}| j!||t|ƒ|
| j
d}| j "| j#||||
|d¡ n d| jkrÂ| j$|||
||d dS )zã
        creates jobs based on scheduler
        :param batch_file: batch job name
        :param email_notif: If email notifications should be on or not. Defaults to true.
        :return: True if running on a cluster
        Nr>   é   Úbatchrh   zMjob_submission.py --template {t} {a} --outdir {b} --numBursts {c} --writeonly)ÚtÚaÚbÚcz --tmpz --distributeú {}ÚrZ
singleTask©r   ZmultiTask_multiNodeZ_0©Únumber_of_tasksrk   r9   ©Ú
batch_filerk   r   ZmultiTask_singleNode)r   Únum_cores_per_task)%r	   rW   rX   rK   r   r=   r   rq   r]   r   r   r    Ú
isinstanceÚlistr   Úlogr9   ÚopenÚ	readlinesÚlenÚnpr'   ÚceilÚfloatrV   rN   rO   rS   rJ   Úwrite_batch_singletask_jobsr0   r3   r7   Úget_job_file_linesÚappendÚadd_tasks_to_job_file_linesÚ
split_jobs)r_   r…   rW   r   r†   Zlog_argsÚitemÚfÚtasksrƒ   rk   Úbatch_file_namert   Újob_file_linesr*   r*   r+   Úwrite_batch_jobs¾   sh    
 ý

ÿ
 ÿý
ÿzJOB_SUBMIT.write_batch_jobsc              	   C   s’   t | jƒdkr$| j| j| jd dS tdƒ t d¡}t|dƒD}| ¡ }| j	dkrZd}nd	}|D ]}t 
|| ¡ |tjd< qbW 5 Q R X d
S dS )zâ
        submit jobs based on scheduler
        :param batch_file: batch job name
        :param email_notif: If email notifications should be on or not. Defaults to true.
        :return: True if running on a cluster
        r   rl   Tz!
Working on a single machine ...
ÚPATHr€   r>   z-export PATH=$ISCE_STACK/stripmapStack:$PATH; z)export PATH=$ISCE_STACK/topsStack:$PATH; FN)rŒ   rX   rs   r   r\   r0   r1   rŠ   r‹   r=   ÚsystemÚenviron)r_   r…   Úsystem_pathr–   Zcommand_linesÚcmdrw   r*   r*   r+   Úsubmit_batch_jobs  s    

zJOB_SUBMIT.submit_batch_jobsc                 C   s  d}| j dkr"dtj ||¡ }nš| j dkr@dtj ||¡ }n|| j dkr¬tjddtjdj ¡  	d	¡}| 
d
¡s|| 
d¡r’d tj ||¡¡}q¼d tj ||¡¡}d}ntd | j ¡ƒ‚tj|tjdd}|rüt d| 	d	¡¡}ttdd„ |D ƒƒƒ}nd}|S )zí
        Submit a single job (to bsub or qsub). Used by submit_jobs_individually and submit_job_with_launcher and submit_script.
        :param job_file_name: Name of job file to submit.
        :return: Job number of submission
        TÚLSFzbsub < ÚPBSzqsub < ÚSLURMÚhostname©ÚshellÚstdoutúutf-8ÚloginÚcometz	sbatch {}z{}Fú"ERROR: scheduler {0} not supported)Ústderrr¦   z\d+c                 S   s   g | ]}t |ƒ‘qS r*   )r'   ©rd   Úxr*   r*   r+   Ú
<listcomp>B  s     z0JOB_SUBMIT.submit_single_job.<locals>.<listcomp>ÚNone)rL   r0   r3   r5   Ú
subprocessÚPopenÚPIPEr§   ÚreadÚdecodeÚ
startswithr]   Ú	ExceptionÚcheck_outputÚSTDOUTÚreÚfindallr&   Úmax)r_   ru   r9   Zjob_num_existsÚcommandr¤   Z
output_jobÚ
job_numberr*   r*   r+   Úsubmit_single_job'  s$    


zJOB_SUBMIT.submit_single_jobr   c           
   	   C   sÄ   t jddt jdj ¡  d¡}| j||||d}| d¡ | jdkrV| j	||||d}| j
rz| d	¡ | d
| d ¡ n| d| d ¡ d |¡}ttj ||¡dƒ}	|	 |¡ W 5 Q R X dS )a  
        Writes a job file for a single job.
        :param job_name: Name of job.
        :param job_file_name: Name of job file.
        :param command_line: Command line containing process to run.
        :param work_dir: working or output directory
        r¤   Tr¥   r¨   rj   z
free
r£   r   z
module load remoraz
remora Ú
ú{0}.jobúw+N)r±   r²   r³   r§   r´   rµ   r‘   r’   rL   Úadd_slurm_commandsr   r]   rŠ   r0   r3   r5   Ú
writelines)
r_   rt   ru   rw   r9   rk   r   r¤   r™   Újob_filer*   r*   r+   rr   H  s    	
ÿ



z JOB_SUBMIT.write_single_job_filec           
   	   C   sŒ   t |ƒ}| ¡ }W 5 Q R X t|ƒD ]b\}}tj |¡ tj¡d d t|ƒ }| j	|||tj 
|¡||d tj 
|¡d | d }	q$dS )ab  
        Iterates through jobs in input file and writes a job file for each job using the specified scheduler. This function
        is used for batch jobs in pegasus (LSF) to split the tasks into multiple jobs
        :param batch_file: File containing batch of jobs for which we are creating job files.
        :return: List of job file names.
        éÿÿÿÿÚ_)r9   rk   r   r-   z.jobN)rŠ   r‹   Ú	enumerater0   r3   r4   r8   Úsepr&   rr   rZ   )
r_   r…   rk   r   Ú
input_fileZjob_listÚirw   ru   rÅ   r*   r*   r+   r   i  s    
$ ÿz&JOB_SUBMIT.write_batch_singletask_jobsc              
   C   sZ  g }g }g }|D ]ˆ}t  d t j ||¡¡¡ |  ||¡}t j |d | d¡d |¡¡}t j |d | d¡d |¡¡}	| |¡ | |¡ | |	¡ qd}
d}d}t 	d¡ | j
dkrŒg }t j |d	¡}t||ƒD ]\}}|d
ksØd}|dkrØt  d ||¡¡ t 	d¡ t|dƒ(}| ¡ }t|ƒdk r@W 5 Q R £ qîW 5 Q R X d|d ksfd|d krštd ||¡ƒ ||d 7 }t 	|d ¡ |
d7 }
qîd|d kr®d}qîd|d krÌd}| |¡ qîd}td |¡ƒ‚qîqØt|ƒdkrä|D ]„}t |¡}tj|dd}t ||¡ t t ¡ d¡}|d t j |¡ d | d | }t| jd d ƒ}| |¡ W 5 Q R X qô| j|| jd! nXt||ƒD ]L\}}d
|kr–t j |¡s–td ||¡ƒ ||d 7 }t 	|¡ q¨q–|D ]l}t | d¡d d" ¡}|D ]H}t|d#ƒt|d$ƒt|d%ƒt|d&ƒg}t  !|¡ "¡ rtd' |¡ƒ‚qqèd(S ))aƒ  
        Writes a single job file for launcher to submit as array. This is used to submit jobs in slurm or sge where launcher
        is available (compare to submit_jobs_individually used on pegasus with LSF)
        :return:
        :param batch_file: File containing tasks that we are submitting.
        :param work_dir: the directory to check outputs and error files of job
        zchmod +x {}z{}_{}.orC   r   z{}_{}.eé<   é   r£   Z
job_statusr°   Úwaitz!sacct --format="State" -j {} > {}r€   é   ZPENDINGZRUNNINGz/Waiting for job {} output file after {} minutesr   Z	COMPLETEDÚcompleteZTIMEOUTÚtimeoutÚfailedz'Error: {} job was terminated with Errorg333333ó?)Úfactorz%Y%m%d:%H-%Mz: re-running: z: z --> z
/rerun.logr|   rl   z*.ezSegmentation faultZAbortedÚERRORÚErrorzError terminating job: {}N)#r0   rœ   r]   r3   r5   r¿   r8   r’   ÚtimeÚsleeprL   ÚziprŠ   r‹   rŒ   r\   ÚRuntimeErrorÚputilsÚextract_walltime_from_job_fileÚmultiply_walltimeÚreplace_walltime_in_job_filer   ÚstrftimeÚnowr7   r   rÄ   rs   r9   ÚexistsrY   Úcheck_words_in_filer   ÚarrayÚany)r_   rX   r9   Zjob_numbersZjobs_outZjobs_errru   Zjob_numÚoutÚerrrË   Zwait_time_secZtotal_wait_time_minZrerun_job_filesZjob_status_filer¾   Zjob_statZ	stat_fileÚstatusr   Únew_wall_timeÚdateStrÚstringZrerunÚerror_filesÚerrfileZjob_exitr*   r*   r+   rs   |  sŠ    	  



ÿ

$
ýz&JOB_SUBMIT.submit_and_check_job_statusc                    sÐ  |}d}| j }d|ksd|kr"d}d| jkr0d}|t|ƒkrT|d }t || ¡}q0tt t|ƒ| ¡ƒ‰ t| j| | j ƒ}	|	ˆ k r¶|t|ƒk r¶|d7 }tt t|ƒ| ¡ƒ‰ q~q¶q~|	ˆ k rÜ|d }t| j| | j ƒ}	q¶|dkròtd 	|¡ƒ tj
dt|ƒˆ …  ¡ }
‡ fd	d
„|
D ƒ}t|ƒ|d< |dk	r@| j| | _nt ˆ | ¡| _t|
|ƒD ]p\}}|
 |¡}|d 	|¡ }tj |¡}| j|||| || jd}| j||||… |||d}| j |¡ qZdS )zÝ
        splits the batch file tasks into multiple jobs with one node
        :param batch_file:
        :param tasks:
        :param number_of_nodes: Total number of nodes required for all tasks
        :return:
        r   Zgenerate_burst_igramZmerge_burst_igraméd   Z
singleNodeiè  z|Note: Number of jobs exceed the numbers allowed per queue for jobs with 1 node...
Number of Nodes per job are adjusted to {}r   c                    s   g | ]}|ˆ  ‘qS r*   r*   r­   ©Znumber_of_parallel_tasksr*   r+   r¯     s     z)JOB_SUBMIT.split_jobs.<locals>.<listcomp>rÆ   Nz_{}r‚   r„   )rP   rJ   r'   r   rŽ   rŒ   rQ   rT   r\   r]   ÚogridÚtolistrN   rS   ÚmathrØ   Úindexr0   r3   r7   r‘   r   r“   rX   r’   )r_   r…   r—   rk   r   r†   Znumber_of_jobsZnumber_of_nodes_per_jobrP   Znumber_of_limited_memory_tasksZstart_linesZ	end_linesZ
start_lineZend_lineZ	job_countr˜   rt   r™   ru   r*   rí   r+   r”   Ö  sZ    	
ÿ

 ÿýzJOB_SUBMIT.split_jobsrz   c                 C   sâ  t jdd}|dkr<d}| | d¡d  d¡dd… ¡}n|}| jd	krj|dkrj| jdkrjt  | j¡| _| jrx| j}nd
}| jdkrÒ||kr¨|| d }|| d }n|d d }|d d }t  	|||¡| _
n| j| _
| jdkrj||kr || d }|| d }	t|| d ƒ| j }
n.|d d }|d d }	t|d d ƒ| j }
t  || j||	|
| j¡| _n
| j}d}	t  || j||	|
| j¡| _||kr¨|| d | _n|d d | _||krÐ|| d | _n|d d | _dS )z¶
        get memory, walltime and number of threads for the job from job_defaults.cfg
        :param job_name: the job file name
        :param job_type: 'batch' or 'script'
        zjob_defaults.cfg)Úconfig_filerz   rÇ   r-   rÆ   rÍ   NrB   r   )Nr°   Úc_memoryÚs_memoryr   Ú
c_walltimeÚ
s_walltimeÚextra_secondsÚ0Únum_threadsr    )rÚ   Úget_config_defaultsr5   r8   r=   r   Úget_number_of_burstsr^   r   Úscale_memoryrT   r   r   r   Úscale_walltimerR   rL   rU   rV   Úcopy_to_tmp_flag)r_   rt   ri   ÚconfigÚ	step_nameÚnumber_of_burstsró   rô   rõ   rö   r÷   r*   r*   r+   rq     s`    $



   ÿ
   ÿ

zJOB_SUBMIT.get_memory_walltimec                 C   sê  | j dkrD|}d}d}d}d}	d| d }
d}d	}d
}d}d}d}n¢| j dkr„d}d}d}d}	d}
d}d}d
}d}d}d| d }nb| j dkrÖ|| j }d}d}d}d}	d| d }
d}d	}d}d| d }d}d }ntd! | j ¡ƒ‚| jd"krø|d#9 }d$| || tj |¡¡ ||	 t d%¡¡ g}| j	rJ| 
|| t d&¡¡ ¡ | ||
 t|ƒt|ƒ¡ || tj ||¡¡ || tj ||¡¡ || | j¡ || | j¡ g¡ | j dkrÌ| 
|d' ¡ | jd(kræ| 
|d) ¡ |S )*aÇ  
        Generates the lines of a job submission file that are based on the specified scheduler.
        :param job_name: Name of job.
        :param job_file_name: Name of job file.
        :param number_of_tasks: Number of lines in batch file to be supposed as number of tasks
        :param number_of_nodes: Number of nodes based on number of tasks (each node is able to perform 68 tasks)
        :return: List of lines for job submission file
        r¡   z
#BSUB z	/bin/bashz-J {0}z-P {0}z-n {0}z-R span[hosts={1}]z-o {0}_%J.oz-e {0}_%J.ez-q {0}z-W {0}z-M {}z	-B -u {0}r¢   z
#PBS z-N {0}z-A {0}z-l nodes={0}:ppn={1}z-o {0}_$PBS_JOBID.oz-e {0}_$PBS_JOBID.ez-l walltime={0}z
-l mem={0}z-m az-M {0}r£   z	
#SBATCH z-n {1}z-p {0}z--mail-user={}z--mail-type=failz-t {0}Fr«   Úparallelé   z#! ZJOBSHEDULER_PROJECTNAMEZNOTIFICATIONEMAILz-VZgpuz--gres=gpu:4)rL   rN   r·   r]   r   r0   r3   r7   r1   rW   r’   Úextendr'   r5   rU   )r_   rt   ru   rƒ   rk   r9   r=   r¦   Zname_optionZproject_optionZprocess_optionZstdout_optionZstderr_optionZqueue_optionZwalltime_limit_optionZmemory_optionZemail_optionr™   r*   r*   r+   r‘   c  sv    




ýûzJOB_SUBMIT.get_job_file_linesc                 C   s  | j r| jdkr|S | d¡ | d¡ | d¡ | d¡ | d || j¡¡ |d k	r®d ||| j¡}t|tƒrŒ|D ]}|d |¡7 }qvn|d |¡7 }|d7 }| |¡ n| d ||| j¡¡ | d	¡ | d
¡ | d¡ | d¡ | jdkr| d¡ n
| d¡ |S )NÚnorÀ   ú1################################################
z1#   install code on /tmp                       #
z(install_code_on_tmp.bash {} --prefix {}
zcopy_data_to_tmp.bash {} {} {}r   zcopy_data_to_tmp.bash {} {} {}
z# set environment    
zexport PATH=/bin
zunset PYTHONPATH
z(export RSMASINSAR_HOME=/tmp/rsmas_insar
r>   z’cd $RSMASINSAR_HOME; source ~/accounts/platforms_defaults.bash; source setup/environment.bash; export PATH=$ISCE_STACK/stripmapStack:$PATH; cd -;
zŽcd $RSMASINSAR_HOME; source ~/accounts/platforms_defaults.bash; source setup/environment.bash; export PATH=$ISCE_STACK/topsStack:$PATH; cd -;
)r    rþ   r’   r]   r=   r   r‡   rˆ   )r_   r™   ru   r¤   r…   r   Zjob_cmdr•   r*   r*   r+   rÃ   ¿  s6    



$





   5zJOB_SUBMIT.add_slurm_commandsc                 C   sö  d}| j dkrDtjddtjdj ¡  d¡}| d¡r@| d¡sDd}d	 |¡}g }	d
| j	ksb|r|D ]^}
t
 |
¡}t
 |¡}|	 d |
 d¡d tj |¡d | d tj |¡d | d ¡¡ qftj |¡rÜt |¡ t|dƒ}| |	¡ W 5 Q R X | j dkr| j|||||d}| d¡ | d¡ | d¡ | d | j¡¡ | d | j¡¡ | d |¡¡ | d |¡¡ | d¡ | d¡ | jrÊ| d¡ | d¡ | dtj |¡ d ¡ n
| d ¡ | d!¡ ttj | j|¡dƒ}| |¡ W 5 Q R X næt|ƒD ]L\}}
|	 d" |
 d¡d tj |¡d# |¡ tj |¡d$ |¡ ¡¡ q| j dkrx| d%¡ | d& | j¡¡ | d' | j¡¡ | d¡ |	D ]}
| |
¡ qª| d(¡ ttj | j|¡dƒ}| |¡ W 5 Q R X |S ))a^  
        complete job file lines based on job submission scheme. if it uses launcher, add launcher specific lines
        :param job_file_lines: raw job file lines from function 'get_job_file_lines'
        :param tasks:number of tasks to be include in this job
        :param batch_file: name of batch file containing tasks
        :return:
        Fr£   r¤   Tr¥   r¨   r©   rª   rÁ   Zlauncherz{} > {} 2>{}
rÀ   r   rÇ   z_$LAUNCHER_JID.oz_$LAUNCHER_JID.erÂ   )r…   r   z2
################################################
z# execute tasks with launcher
r  zexport OMP_NUM_THREADS={0}
zexport LAUNCHER_PPN={0}
zexport LAUNCHER_NHOSTS={0}
zexport LAUNCHER_JOB_FILE={0}
z!export LAUNCHER_WORKDIR=/dev/shm
zcd /dev/shm
z

module load remoraz
remora $LAUNCHER_DIR/paramrun
z 
mv remora_$SLURM_JOB_ID remora_z_$SLURM_JOB_ID
z$LAUNCHER_DIR/paramrun
zrm -rf /tmp/rsmas_insar 
z{} > {} 2>{} &
z_{}.oz_{}.ezC
export LD_PRELOAD=/home1/apps/tacc-patches/python_cacher/myopen.soz

export OMP_NUM_THREADS={0}z
export PATH={0}:$PATHz
wait)rL   r±   r²   r³   r§   r´   rµ   r¶   r]   rJ   rÚ   Ú$extract_config_file_from_task_stringÚ)extract_date_string_from_config_file_namer’   r8   r0   r3   r4   rà   ro   rŠ   rÄ   rÃ   rV   rS   r   r7   r5   r   rÈ   rH   )r_   r™   r—   r…   rk   r   Zdo_launcherr¤   ru   Ztasks_with_outputÚlinerò   Údate_stringZbatch_fZjob_fÚcountr*   r*   r+   r“   ¶  sr    



þ

 ÿ









þ



z&JOB_SUBMIT.add_tasks_to_job_file_lines)Nrb   )NNNN)N)Nr   N)r   N)N)NN)rz   )r   r   N)NN)Nr   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ra   rx   rš   r    r¿   rr   r   rs   r”   rq   r‘   rÃ   r“   r*   r*   r*   r+   r<   i   s"   1

M
!
!

Z
H
E
\
   zr<   c              	      sN   t | dƒ}| ¡ }W 5 Q R X ‡ fdd„|D ƒ}t d| ¡dkrFdS dS dS )	zÈ
    Checks for existence of a specific word in a file
    :param errfile: The file to be checked
    :param eword: The word to search for in the file
    :return: True if the word is in the file
    r€   c                    s   g | ]}ˆ |k‘qS r*   r*   )rd   r•   ©Úewordr*   r+   r¯      s     z'check_words_in_file.<locals>.<listcomp>r   r   TFN)rŠ   r‹   r   Úsum)rë   r  r–   ÚlinesZcheck_ewordr*   r  r+   rá     s    rá   c              
   C   sV  t | ƒ}|d }|dkr2t d¡r.t d¡}nd}tjddtjdj ¡  d¡}t	D ]}||krj|} qpqTd }qT| j
r‚| j
|d	< nt d	¡ršt d	¡|d	< |d	 |d
 |d |d |d |d |d dœ}| ¡ D ]B}|| dksÒ|dkrüt|| ƒ||< qÒ|dksÒt|| ƒ||< qÒ|t	kr¦ttdƒ}| ¡ }	W 5 Q R X |	D ]$}
|
 d¡rB|	d  ¡ } qhqB|	D ]6}
|
 d¡sl|
 |¡rl|
 ¡ }|| d	¡ }|d dkr¶||d< ||d krl|d dkrèt|| d
¡ ƒ|d< |d dkrt|| d¡ ƒ|d< |d dkr0t|| d¡ ƒ|d< |d dkrTt|| d¡ ƒ|d< |d dkrxt|| d¡ ƒ|d< |d dkrœt|| d¡ ƒ|d<  q¦ql|dkr¶d}n$|dkrÆd }n|d!krÖd"}nd }d d#d$d$d%d$g}d}| ¡ D ],\}}|| dkrö|| ||< |d$7 }qö||||d |d |d |d |d |d f	}|S )&NÚjob_submission_schemeÚautoZJOB_SUBMISSION_SCHEMEZlauncher_multiTask_singleNodezhostname -fTr¥   r¨   Ú	QUEUENAMEÚCPUS_PER_NODEÚTHREADS_PER_COREÚMAX_JOBS_PER_WORKFLOWÚMAX_JOBS_PER_QUEUEÚWALLTIME_FACTORÚMEM_PER_NODE)rM   rN   rO   rP   Úmax_jobs_per_queuerR   rQ   rR   rM   r€   ZPLATFORM_NAMEr   ú#)r  ÚNONEr°   rN   rO   r  rP   rQ   )Ú	stampede2Úfronterarª   r£   )Úpegasusr¡   )Úeos_sanghoonÚbeijing_serverÚdeqing_serverÚeosÚdqcentos7insarr¢   r  r   i€>  )Ú"auto_template_not_existing_optionsr0   r1   r±   r²   r³   r§   r´   rµ   r   r   rF   r   r'   rŠ   r   r‹   r¶   r8   rñ   Úitems)r:   r[   rJ   r¤   ÚplatformrK   Z
check_autoÚkeyr–   r  r	  Zqueue_headerZsplit_valuesZdefault_queuerL   Zdef_autorË   ÚvalueZout_putsr*   r*   r+   rI   (  sš    

ú






  þrI   c                 C   sl   ddddddddg}t | d	ƒrRd
dlm} || jƒj}|D ]}||kr:d||< q:ni }|D ]}d||< qZ|S )Nr  r  r  r  r  r  r  r  r   r   )ÚTemplater  )ÚhasattrÚminsar.objects.dataset_templater-  r   Úoptions)r:   Zjob_optionsr-  r[   Úoptionr*   r*   r+   r(  ‰  s    
  ÿ

r(  Ú__main__r   r   F))r  r0   ÚsysÚstatr±   r!   rÖ   rY   Únumpyr   rð   ÚtextwrapÚminsar.objectsr   Úminsar.objects.auto_defaultsr   r   ÚwarningsZminsar.utils.process_utilitiesÚutilsÚprocess_utilitiesrÚ   r   rº   ÚfilterwarningsÚFutureWarningr,   r;   r<   rá   rI   r(  r  rv   ZPARAMSÚjob_objrš   r   r   r    ræ   r*   r*   r*   r+   Ú<module>   sL   !         5a
