o QpÆeÉã@s0dZddlZddlZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl m Z ddlmZmZddlZddlmmZddlmZddlZejdeddd „Zd d „ZGd d „d ƒZdd„Zdd„Zdd„Ze dkr”eej!dd…ƒZ"ee"ƒZ#e#j$e"j%de"j&dur–e# '¡Z(dSdSdS)a‡ Functions related to batch job submission. Should be run with a file containing jobs to submit as a batch. Optional parameters for job submission are, --memory, --walltime, and --queuename. Generates job scripts, runs them, and waits for output files to be written before exiting. This script has functions to support submitting two different job types: a script as a job or a batch file consisting of multiple parallel tasks. submitting a script as a job is done calling the function: submit_script However submitting a batch file (calling submit_batch_jobs) can be done in 5 different ways with or without launcher. Two environmental variables have to be set: JOB_SUBMISSION_SCHEME and QUEUENAME (set in '~/accounts/platforms_defaults.bash') QUEUENAME has defaults based on platforms. comment/uncomment or introduce a new one JOB_SUBMISSION_SCHEME: it can have one of these options: singleTask ---> submit each task of a batch file separately in a job multiTask_singleNode ---> distribute tasks of a batch file into jobs with one node multiTask_multiNode ---> submit tasks of a batch file in one job with required number of nodes launcher_multiTask_singleNode ---> distribute tasks of a batch file into jobs with one node, submit with launcher launcher_multiTask_multiNode ---> submit tasks of a batch file in one job with required number of nodes using launcher éN)Ú message_rsmas)Úqueue_config_fileÚsupported_platforms)ÚdatetimeÚignore)ÚcategorycCstjtjd}| dd¡}|jdtdd|jddtd d d |jd d ddd|jddddd|jddddd|jdddddd|jdd td!d"d |jd#d$td%d&d'd(|jd)d*td+d,d-d.|jd/d0d1d2d3|jd4d5d1d6d3|jd7d8d1d9d3|S):a Creates an argument parser for parsing parameters for batch job submission. Required parameter: file to batch create Optional parameters: template file, memory, walltime, and queue name :return: ArgumentParser object for parsing command line batch job submission )Úformatter_classz Input FilezFile/Dataset to displayÚfilezThe file to batch create)ÚtypeÚhelpz --templateÚcustom_template_filez template filez&custom template with option settings. )Údestr Úmetavarr z--memoryÚmemoryz MEMORY (KB)z4Amount of memory to allocate, specified in kilobytes)r rr z --walltimeÚ wall_timezWALLTIME (HH:MM)z+Amount of wall time to use, in HH:MM formatz--queueÚqueueÚQUEUEzName of queue to submit job toz--outdirÚout_dirÚ run_filesZOUTDIRzoutput directory for run files)r Údefaultrr z--numMemoryUnitsÚnum_memory_unitsznumber of memory unitszxnumber of memory units to calculate walltime, for ISCE workflow:number of bursts, for Miaplpy workflow:number of patchesz --numDataÚnum_dataéznumber of dataz;number of data (interferogram or slc) to calculate walltime)r r rrr z --distributeÚ distributeú+NzFiles to be copied to /tmp)r r Únargsrr z --writeonlyÚ writeonlyÚ store_truez"Write job files without submitting)r Úactionr z--remoraÚremoraz!use remora to get job informationz--tmpÚ copy_to_tmpz!write to run_files_tmp/ directory)ÚargparseÚArgumentParserÚRawTextHelpFormatterÚadd_argument_groupÚ add_argumentÚstrÚint)ÚparserÚgroup©r*ú=/home/exouser/operations/rsmas_insar/minsar/job_submission.pyÚcreate_argument_parser.s:  ÿ ÿ ÿ ÿ ÿÿÿr,cCstƒ}| |¡}zt d¡}Wnt ¡}Ytj |j¡|_tj ||j  tj  |¡¡d  d¡d¡|_ d|j vrFtj |j |j ¡|_ |S)z· Parses command line arguments into namespace. :param args: Arguments to parse :return: Namespace with submission parameters (from command line arguments or defaults) ZSCRATCHrú/r)r,Ú parse_argsÚosÚgetenvÚgetcwdÚpathÚabspathr ÚjoinÚrsplitÚbasenameÚsplitÚwork_dirr)Úargsr(Z job_paramsZ scratch_dirr*r*r+Úparse_argumentsOs   ÿ r:c@sŽeZdZdZdd„Zd!dd„Zd"dd „Zd#d d „Zd d „Zd$dd„Z d%dd„Z d&dd„Z d'dd„Z d(dd„Z d)dd„Zd'dd„Zd$dd „ZdS)*Ú JOB_SUBMITz< A class representing the job submission object c Cs®|j ¡D] }t|||j|ƒqd|vr)|jdkr)tj t d¡d¡|_d|_ntj t d¡d¡|_d|_t |ƒ\ |_ |_ |_ |_ |_|_|_|_|_d|_d|vsW|jsZd|_d |vsa|jsdd|_d |vsk|jsnd|_d |vsu|jsy|j |_d |vr€d |_d|vr‡d|_d|vrŽd|_|j|_d|_d|_d|_d|vs¢|js¥d|_d|_ g|_!zt" "|j#d¡d}tj $|¡|j%|jd<Wnt&d '|j#d¡ƒY||_(dS)NÚprefixÚstripmapÚ ISCE_STACKÚ stripmapStackÚ topsStackÚtopsrrrrrrÚ.rr Ú reserve_nodeTz /DEM/*.wgs84rz Stack.demDirzDEM does not exist in {}z/DEM))Ú__dict__ÚkeysÚsetattrr<r/r2r4r0Ú stack_pathÚset_job_queue_valuesÚsubmission_schemeÚ platform_nameÚ schedulerÚ queue_nameÚnumber_of_cores_per_nodeÚnumber_of_threads_per_coreÚmax_jobs_per_workflowÚmax_memory_per_nodeÚwall_time_factorÚ!number_of_parallel_tasks_per_noderrrrrrr rÚdefault_memoryÚdefault_wall_timeÚdefault_num_threadsrCÚ email_notifÚ job_filesÚglobr8ÚdirnameÚtemplateÚprintÚformatÚinps)Úselfr]ÚkZdem_filer*r*r+Ú__init__osPþ  zJOB_SUBMIT.__init__NÚFalsecCs¾tj |j¡stj |j¡rt |j¡t |j¡|dur!||_d dd„|dd…Dƒ¡}|j |ddtj |jd  |¡¡g|_ |j ||||j|j d |d kr]|j|j |jd d dS) aŸ Submits a single script as a job. (compare to submit_batch_jobs for several tasks given in run_file) :param job_name: Name of job. :param job_file_name: Name of job file. :param argv: Command line arguments for running job. :param email_notif: If email notifications should be on or not. Defaults to true. :return job number of the script that was submitted Nú css|] }|dkr|VqdS)z--submitNr*)Ú.0Úflagr*r*r+Ú ²s€z+JOB_SUBMIT.submit_script..rÚscript©Újob_typeú{0}.job©r8Únumber_of_nodesraF)r8Ú wait_flag)r/r2Úisdirr8ÚisfileÚremoveÚmakedirsrVr4Úget_memory_walltimer\rWÚwrite_single_job_filerCÚsubmit_and_check_job_status)r^Újob_nameÚ job_file_nameÚargvrVZ writeOnlyÚ command_liner*r*r+Ú submit_script s   ÿzJOB_SUBMIT.submit_scriptc CsÒ|dur|j}|dur||_g|_|jtvrç|jdkrd|_|j|dddj|j ||j |jd}|j r8|d7}|durY|d 7}t |t ƒrR|D] }|d  |¡7}qGn|d  |¡7}t |j|¡t|d ƒ}| ¡}t|ƒ} Wdƒn1sxwYtt | t|jƒ|j|j¡ƒ} |dur|j||_| |7} d |jvr¬|j|| |d dSd|jvrØ|d} tj  | ¡} |j!|| t|ƒ| |j d} |j "|j#| || | |d¡dSd|jvrç|j$||| ||ddS)zã creates jobs based on scheduler :param batch_file: batch job name :param email_notif: If email notifications should be on or not. Defaults to true. :return: True if running on a cluster Nr=éÚbatchrgzRjob_submission.py --template {t} {a} --outdir {b} --numMemoryUnits {c} --writeonly)ÚtÚaÚbÚcz --tmpz --distributeú {}ÚrZ singleTask©rZmultiTask_multiNodeZ_0©Únumber_of_tasksrkr8©Ú batch_filerkrZmultiTask_singleNode)rÚnum_cores_per_task)%r rVrWrJrr<rrqr\r rr Ú isinstanceÚlistrÚlogr8ÚopenÚ readlinesÚlenr'ÚnpÚceilÚfloatrUrMrNrRrIÚwrite_batch_singletask_jobsr/r2r6Úget_job_file_linesÚappendÚadd_tasks_to_job_file_linesÚ split_jobs)r^r…rVrr†Zlog_argsÚitemÚfÚtasksrƒrkÚbatch_file_namertÚjob_file_linesr*r*r+Úwrite_batch_jobs¿sl  ý ÿ  þ ÿ   ï ÿý û ÿzJOB_SUBMIT.write_batch_jobscCs¤t|jƒdkr|j|j|jddStdƒt d¡}t|dƒ(}| ¡}|j dkr-d}nd }|D]}t  ||¡|tj d<q1Wd ƒd S1sKwYd S) zâ submit jobs based on scheduler :param batch_file: batch job name :param email_notif: If email notifications should be on or not. Defaults to true. :return: True if running on a cluster r©r8Tz! Working on a single machine ... ÚPATHr€r=z-export PATH=$ISCE_STACK/stripmapStack:$PATH; z)export PATH=$ISCE_STACK/topsStack:$PATH; NF) rŒrWrsrr[r/r0rŠr‹r<ÚsystemÚenviron)r^r…Ú system_pathr–Z command_linesÚcmdrwr*r*r+Úsubmit_batch_jobs s$    þ ú ö zJOB_SUBMIT.submit_batch_jobscCsd}|jdkrdtj ||¡}nM|jdkr dtj ||¡}n>|jdkrVtjddtjdj ¡  d ¡}|  d ¡s>|  d ¡rId   tj ||¡¡}nd   tj ||¡¡}d}nt d  |j¡ƒ‚tj |tjdd}|rt d|  d ¡¡}ttdd„|Dƒƒƒ}|Sd}|S)zí Submit a single job (to bsub or qsub). Used by submit_jobs_individually and submit_job_with_launcher and submit_script. :param job_file_name: Name of job file to submit. :return: Job number of submission TÚLSFzbsub < ÚPBSzqsub < ÚSLURMÚhostname©ÚshellÚstdoutúutf-8ÚloginÚcometz sbatch {}z{}Fú"ERROR: scheduler {0} not supported)Ústderrr§z\d+cSsg|]}t|ƒ‘qSr*)r'©rcÚxr*r*r+Ú Cóz0JOB_SUBMIT.submit_single_job..ÚNone)rKr/r2r4Ú subprocessÚPopenÚPIPEr¨ÚreadÚdecodeÚ startswithr\Ú ExceptionÚ check_outputÚSTDOUTÚreÚfindallr&Úmax)r^rur8Zjob_num_existsÚcommandr¥Z output_jobÚ job_numberr*r*r+Úsubmit_single_job(s&   þzJOB_SUBMIT.submit_single_jobrc CsÚtjddtjdj ¡ d¡}|j||||d}| d¡|jdkr+|j ||||d}|j r=| d ¡| d |d ¡n | d |d ¡d   |¡}t t j ||¡d ƒ} |  |¡WdƒdS1sfwYdS)a Writes a job file for a single job. :param job_name: Name of job. :param job_file_name: Name of job file. :param command_line: Command line containing process to run. :param work_dir: working or output directory r¥Tr¦r©rjz free r¤rz module load remoraz remora Ú riúw+N)r³r´rµr¨r¶r·r‘r’rKÚadd_slurm_commandsrr\rŠr/r2r4Ú writelines) r^rtrurwr8rkrr¥r™Újob_filer*r*r+rrIs$ ÿ      ÿýz JOB_SUBMIT.write_single_job_filec Cs t|ƒ }| ¡}Wdƒn1swYt|ƒD]1\}}tj |¡ tj¡ddt|ƒ}|j |||tj  |¡||dtj  |¡d|d} qdS)ab Iterates through jobs in input file and writes a job file for each job using the specified scheduler. This function is used for batch jobs in pegasus (LSF) to split the tasks into multiple jobs :param batch_file: File containing batch of jobs for which we are creating job files. :return: List of job file names. NéÿÿÿÿÚ_)r8rkrr-z.job) rŠr‹Ú enumerater/r2r3r7Úsepr&rrrY) r^r…rkrÚ input_fileZjob_listÚirwrurÆr*r*r+rjs  ÿ$ÿz&JOB_SUBMIT.write_batch_singletask_jobsTc Cs’g}g}g}|D]D}t d tj ||¡¡¡| ||¡}tj |d | d¡d|¡¡} tj |d | d¡d|¡¡} | |¡| | ¡| | ¡q|rÇd} d} d} t  d¡|j dkr]g}tj |d ¡}t ||ƒD]\}}|d ksüd }|d krüt d  ||¡¡t  d¡t |d ƒ}|  ¡}t|ƒdkr£ WdƒqyWdƒn1s­wYd|dvs¾d|dvrØtd || ¡ƒ| | d7} t  | d¡| d7} n d|dvrád}nd|dvrïd}| |¡n d}td |¡ƒ‚|d ks}qot|ƒdkr\|D]M}t |¡}tj|dd}t ||¡t t ¡d¡}|dtj |¡d|d|}t |jd d!ƒ }| |¡Wdƒn 1sMwYq|j||jd"n1t ||ƒD]+\} }d | vrŒtj | ¡sŒtd || ¡ƒ| | d7} t  | ¡tj | ¡rrqb|D]6}t | d¡dd#¡}|D]$}t|d$ƒt|d%ƒt|d&ƒt|d'ƒg}t  !|¡ "¡rÃtd( |¡ƒ‚q qdS))aƒ Writes a single job file for launcher to submit as array. This is used to submit jobs in slurm or sge where launcher is available (compare to submit_jobs_individually used on pegasus with LSF) :return: :param batch_file: File containing tasks that we are submitting. :param work_dir: the directory to check outputs and error files of job z chmod +x {}z{}_{}.orBrz{}_{}.eé<ér¤Z job_statusr²Úwaitz!sacct --format="State" -j {} > {}r€éNZPENDINGZRUNNINGz/Waiting for job {} output file after {} minutesrZ COMPLETEDÚcompleteZTIMEOUTÚtimeoutÚfailedz'Error: {} job was terminated with Errorg333333ó?)Úfactorz %Y%m%d:%H-%Mz: re-running: z: z --> z /rerun.logr|r›z*.ezSegmentation faultZAbortedÚERRORÚErrorzError terminating job: {})#r/rr\r2r4rÁr7r’ÚtimeÚsleeprKÚziprŠr‹rŒr[Ú RuntimeErrorÚputilsÚextract_walltime_from_job_fileÚmultiply_walltimeÚreplace_walltime_in_job_filerÚstrftimeÚnowr6rrÅrsr8ÚexistsrXÚcheck_words_in_filerÚarrayÚany)r^rWr8rlZ job_numbersZjobs_outZjobs_errruZjob_numÚoutÚerrrÌZ wait_time_secZtotal_wait_time_minZrerun_job_filesZjob_status_filerÀZjob_statZ stat_fileÚstatusrÚ new_wall_timeÚdateStrÚstringZrerunÚ error_filesÚerrfileZjob_exitr*r*r+rs}s¤            ýþÿ     ì€  $ ÿ€€   ý€ýÿûz&JOB_SUBMIT.submit_and_check_job_statuscsâ|}d}|j}d|vsd|vrd}d|jvrd}|t|ƒkr/|d}t ||¡}|t|ƒkstt t|ƒ|¡ƒ‰t|j||jƒ} | ˆkrc|t|ƒkr^|d7}tt t|ƒ|¡ƒ‰nn| ˆksH| ˆkry|d}t|j||jƒ} | ˆksg|dkr„td  |¡ƒtj dt|ƒˆ…  ¡} ‡fd d „| Dƒ} t|ƒ| d <|d urª|j ||_ nt ˆ|¡|_ t| | ƒD]7\} } |  | ¡}|d   |¡}tj |¡}|j||| | ||jd}|j||| | …|||d}|j |¡q·d S)zÝ splits the batch file tasks into multiple jobs with one node :param batch_file: :param tasks: :param number_of_nodes: Total number of nodes required for all tasks :return: rZgenerate_burst_igramZmerge_burst_igramédZ singleNodeièz|Note: Number of jobs exceed the numbers allowed per queue for jobs with 1 node... Number of Nodes per job are adjusted to {}rcsg|]}|ˆ‘qSr*r*r®©Znumber_of_parallel_tasksr*r+r°r±z)JOB_SUBMIT.split_jobs..rÇNz_{}r‚r„)rOrIr'rrŽrŒrPrSr[r\ÚogridÚtolistrMrRÚmathrÙÚindexr/r2r6r‘rr“rWr’)r^r…r—rkrr†Znumber_of_jobsZnumber_of_nodes_per_jobrOZnumber_of_limited_memory_tasksZ start_linesZ end_linesZ start_lineZend_lineZ job_countr˜rtr™rur*rîr+r”Øs^    þ úþÿ   ÿýzJOB_SUBMIT.split_jobsrzc CsÒtjdd}|dkrd}| | d¡d d¡dd…¡}n|}|jd kr5|dkr5|jdur5t |j¡|_|jr<|j}nd }|jd vri||vrT||d }||d }n |dd }|dd }t  |||¡|_ n|j|_ |j d vr¬||vrŽ||d}||d} t ||dƒ|j } n|dd}|dd} t |ddƒ|j } | dkr«d } n|j }d} d } t ||j|| | |j¡|_||vrÌ||d|_n|dd|_||vrà||d|_dS|dd|_dS)z¶ get memory, walltime and number of threads for the job from job_defaults.cfg :param job_name: the job file name :param job_type: 'batch' or 'script' zjob_defaults.cfg)Ú config_filerzrÈr-rÇrÎNrAr)Nr²Úc_memoryÚs_memoryrÚ c_walltimeÚ s_walltimeÚseconds_factorrÚ0Ú num_threadsr )rÛÚget_config_defaultsr4r7r<rÚget_number_of_burstsr]rÚ scale_memoryrSrrrÚscale_walltimerQrKrTrUÚcopy_to_tmp_flag) r^rtrhÚconfigÚ step_nameÚnumber_of_memory_unitsrôrõrör÷Ú extra_secondsr*r*r+rqsX $          €  ÿþzJOB_SUBMIT.get_memory_walltimec Csä|jdkr"|}d}d}d}d} d|d} d} d } d } d }d }d }nQ|jdkrBd}d}d}d} d} d} d} d } d}d}d|d}n1|jdkrk||j}d}d}d}d} d|d} d} d } d} d|d}d}d }ntd! |j¡ƒ‚|jd"kr||d#9}d$||| tj |¡¡||  t d%¡¡g}|j r¤|  || t d&¡¡¡|  ||  t |ƒt |ƒ¡||  tj  ||¡¡||  tj  ||¡¡||  |j¡|| |j¡g¡|jdkrä|  |d'¡|jd(krð|  |d)¡|S)*aÇ Generates the lines of a job submission file that are based on the specified scheduler. :param job_name: Name of job. :param job_file_name: Name of job file. :param number_of_tasks: Number of lines in batch file to be supposed as number of tasks :param number_of_nodes: Number of nodes based on number of tasks (each node is able to perform 68 tasks) :return: List of lines for job submission file r¢z #BSUB z /bin/bashz-J {0}z-P {0}z-n {0}z-R span[hosts={1}]z -o {0}_%J.oz -e {0}_%J.ez-q {0}z-W {0}z-M {}z -B -u {0}r£z #PBS z-N {0}z-A {0}z-l nodes={0}:ppn={1}z-o {0}_$PBS_JOBID.oz-e {0}_$PBS_JOBID.ez-l walltime={0}z -l mem={0}z-m az-M {0}r¤z #SBATCH z-n {1}z-p {0}z--mail-user={}z--mail-type=failz-t {0}Fr¬Úparalleléz#! ZJOBSHEDULER_PROJECTNAMEZNOTIFICATIONEMAILz-VZgpuz --gres=gpu:4)rKrMr¹r\rr/r2r6r0rVr’Úextendr'r4rT)r^rtrurƒrkr8r<r§Z name_optionZproject_optionZprocess_optionZ stdout_optionZ stderr_optionZ queue_optionZwalltime_limit_optionZ memory_optionZ email_optionr™r*r*r+r‘dsv        ýû  zJOB_SUBMIT.get_job_file_linescCs|jr|jdkr |S| d¡| d¡| d¡| d¡| d ||j¡¡|durWd |||j¡}t|tƒrF|D] }|d |¡7}q;n|d |¡7}|d7}| |¡n | d |||j¡¡| d ¡| d ¡| d ¡| d ¡|jd kr‚| d¡|S| d¡|S)NÚnorÂú1################################################ z1# install code on /tmp # z(install_code_on_tmp.bash {} --prefix {} zcopy_data_to_tmp.bash {} {} {}rzcopy_data_to_tmp.bash {} {} {} z# set environment zexport PATH=/bin zunset PYTHONPATH z(export RSMASINSAR_HOME=/tmp/rsmas_insar r=z’cd $RSMASINSAR_HOME; source ~/accounts/platforms_defaults.bash; source setup/environment.bash; export PATH=$ISCE_STACK/stripmapStack:$PATH; cd -; zŽcd $RSMASINSAR_HOME; source ~/accounts/platforms_defaults.bash; source setup/environment.bash; export PATH=$ISCE_STACK/topsStack:$PATH; cd -; )r rÿr’r\r<rr‡rˆ)r^r™rur¥r…rZjob_cmdr•r*r*r+rÄÀsF    $ ÿ       6 Ì4zJOB_SUBMIT.add_slurm_commandsc Cs8d}|jdkr"tjddtjdj ¡ d¡}| d¡r | d¡s"d}d  |¡}g} d |j vs1|r|D]/} t   | ¡} t   | ¡} |   d  |  d ¡d tj |¡d| dtj |¡d| d¡¡q3tj |¡rnt |¡t|dƒ } |  | ¡Wdƒn1sƒwY|jdkr—|j|||||d}|  d¡|  d¡|  d¡|  d |j¡¡|  d |j¡¡|  d |¡¡|  d |¡¡|  d¡|  d¡|jrí|  d¡|  d¡|  dtj |¡d ¡n|  d!¡|  d"¡ttj |j|¡dƒ}| |¡Wdƒ|S1swY|St|ƒD]&\}} |   d# |  d ¡d tj |¡d$ |¡tj |¡d% |¡¡¡q|jdkrQ|  d&¡|  d' |j¡¡|  d( |j¡¡|  d ¡| D]} |  | ¡qj|  d)¡ttj |j|¡dƒ}| |¡Wdƒ|S1s•wY|S)*a^ complete job file lines based on job submission scheme. if it uses launcher, add launcher specific lines :param job_file_lines: raw job file lines from function 'get_job_file_lines' :param tasks:number of tasks to be include in this job :param batch_file: name of batch file containing tasks :return: Fr¤r¥Tr¦r©rªr«riZlauncherz {} > {} 2>{} rÂrrÈz_$LAUNCHER_JID.oz_$LAUNCHER_JID.erÃN)r…rz2 ################################################ z# execute tasks with launcher rzexport OMP_NUM_THREADS={0} zexport LAUNCHER_PPN={0} zexport LAUNCHER_NHOSTS={0} zexport LAUNCHER_JOB_FILE={0} z!export LAUNCHER_WORKDIR=/dev/shm z cd /dev/shm z module load remoraz remora $LAUNCHER_DIR/paramrun z mv remora_$SLURM_JOB_ID remora_z_$SLURM_JOB_ID z$LAUNCHER_DIR/paramrun zrm -rf /tmp/rsmas_insar z{} > {} 2>{} & z_{}.oz_{}.ezC export LD_PRELOAD=/home1/apps/tacc-patches/python_cacher/myopen.soz export OMP_NUM_THREADS={0}z export PATH={0}:$PATHz wait)rKr³r´rµr¨r¶r·r¸r\rIrÛÚ$extract_config_file_from_task_stringÚ)extract_date_string_from_config_file_namer’r7r/r2r3rárorŠrÅrÄrUrRrr6r4rrÉrG)r^r™r—r…rkrZ do_launcherr¥ruZtasks_with_outputÚlineróÚ date_stringZbatch_fZjob_fÚcountr*r*r+r“¶s€    þ    ÿ  ÿ          ÿäê þ      ÿýz&JOB_SUBMIT.add_tasks_to_job_file_lines)Nra)NNNN)N)NrN)rN)NT)NN)rz)rrN)Ú__name__Ú __module__Ú __qualname__Ú__doc__r`rxršr¡rÁrrrrsr”rqr‘rÄr“r*r*r*r+r;js$ 1  M ! !  [ G E \yr;cs^t|dƒ }| ¡}Wdƒn1swY‡fdd„|Dƒ}t d|¡dkr-dSdS) zÈ Checks for existence of a specific word in a file :param errfile: The file to be checked :param eword: The word to search for in the file :return: True if the word is in the file r€Ncsg|]}ˆ|v‘qSr*r*)rcr•©Úewordr*r+r° r±z'check_words_in_file..rrTF)rŠr‹rÚsum)rìrr–ÚlinesZ check_ewordr*rr+râs  ÿrâc CsPt|ƒ}|d}|dkrt d¡rt d¡}nd}tjddtjdj ¡ d¡}t D] }||vr4|}nd}q*|j r@|j |d <n t d ¡rLt d ¡|d <|d |d |d |d |d |d|ddœ}|  ¡D]!}||dks‰|dkr}t ||ƒ||<qh|dks‰t ||ƒ||<qh|t vrTttdƒ }| ¡} Wdƒn1s£wY| D]} |  d¡r¹| d ¡} nqª| D]—} |  d¡sS|  |¡rS|  ¡} | |  d ¡} |ddvrß| |d<| |dkrS|ddkr÷t | |  d ¡ƒ|d<|ddkr t | |  d ¡ƒ|d<|ddkrt | |  d ¡ƒ|d<|ddkr-t | |  d ¡ƒ|d<|ddkr?t | |  d¡ƒ|d<|ddkrQt | |  d¡ƒ|d<nq¼|dvr\d}n|dvrdd }n |d!vrld"}nd}gd#¢}d}| ¡D]\}}||dkr||||<|d$7}qx||||d|d|d|d|d|df }|S)%NÚjob_submission_schemeÚautoZJOB_SUBMISSION_SCHEMEZlauncher_multiTask_singleNodez hostname -fTr¦r©Ú QUEUENAMEÚ CPUS_PER_NODEÚTHREADS_PER_COREÚMAX_JOBS_PER_WORKFLOWÚMAX_JOBS_PER_QUEUEÚWALLTIME_FACTORÚ MEM_PER_NODE)rLrMrNrOÚmax_jobs_per_queuerQrPrQrLr€Z PLATFORM_NAMErú#)rÚNONEr²rMrNrrOrP)Ú stampede3Úfronterar«r¤)Úpegasusr¢)Ú eos_sanghoonÚbeijing_serverÚ deqing_serverÚeosÚdqcentos7insarr£)Nrrri€>rr)Ú"auto_template_not_existing_optionsr/r0r³r´rµr¨r¶r·rrrErr'rŠrr‹r¸r7ròÚitems)r9rZrIr¥ÚplatformrJZ check_autoÚkeyr–rr Z queue_headerZ split_valuesZ default_queuerKZdef_autorÌÚvalueZout_putsr*r*r+rH(s     ú  €   ÿ  þ  €    €  þrHcCsbgd¢}t|dƒr$ddlm}||jƒj}|D] }||vr!d||<q|Si}|D]}d||<q(|S)N)rrrrrrrrr r)ÚTemplater)ÚhasattrÚminsar.objects.dataset_templater/r Úoptions)r9Z job_optionsr/rZÚoptionr*r*r+r*‰s   €ü r*Ú__main__rrF))rr/ÚsysÚstatr³r!r×rXÚnumpyrrñÚtextwrapÚminsar.objectsrÚminsar.objects.auto_defaultsrrÚwarningsÚminsar.utils.process_utilitiesÚutilsÚprocess_utilitiesrÛrr¼ÚfilterwarningsÚ FutureWarningr,r:r;rârHr*rrvZPARAMSÚjob_objršrrr¡rçr*r*r*r+ÚsR  !4a  ú