r"""
# send in job name and kwargs for slurm params:
>>> s = Slurm("job-name", {"account": "ucgd-kp", "partition": "ucgd-kp"})
>>> print(str(s))
#!/bin/bash
<BLANKLINE>
#SBATCH -e logs/job-name.%J.err
#SBATCH -o logs/job-name.%J.out
#SBATCH -J job-name
<BLANKLINE>
#SBATCH --account=ucgd-kp
#SBATCH --partition=ucgd-kp
<BLANKLINE>
set -eo pipefail -o nounset
<BLANKLINE>
{script}

>>> s.run("do stuff", _cmd="ls", name_addition="")

"""
from __future__ import print_function

import sys
import os
import subprocess
import tempfile
import atexit
import hashlib
import datetime

TMPL = """\
#!/bin/bash

#SBATCH -e logs/{name}.%J.err
#SBATCH -o logs/{name}.%J.out
#SBATCH -J {name}

{header}

set -eo pipefail -o nounset

{script}"""


def tmp(suffix=".sh"):
    t = tempfile.mktemp(suffix=suffix)
    atexit.register(os.unlink, t)
    return t


class Slurm(object):
    def __init__(self, name, slurm_kwargs=None, tmpl=None, date_in_name=True, scripts_dir="scripts/"):
        if slurm_kwargs is None:
            slurm_kwargs = {}
        if tmpl is None:
            tmpl = TMPL

        header = []
        for k, v in slurm_kwargs.items():
            if len(k) > 1:
                k = "--" + k + "="
            else:
                k = "-" + k + " "
            header.append("#SBATCH %s%s" % (k, v))
        self.header = "\n".join(header)
        self.name = "".join(x for x in name.replace(" ", "-") if x.isalnum() or x == "-")
        self.tmpl = tmpl
        self.slurm_kwargs = slurm_kwargs
        if scripts_dir is not None:
            self.scripts_dir = os.path.abspath(scripts_dir)
        else:
            self.scripts_dir = None
        self.date_in_name = bool(date_in_name)


    def __str__(self):
        return self.tmpl.format(name=self.name, header=self.header,
                                script="{script}")

    def _tmpfile(self):
        if self.scripts_dir is None:
            return tmp()
        else:
            if not os.path.exists(self.scripts_dir):
                os.makedirs(self.scripts_dir)
            return "%s/%s.sh" % (self.scripts_dir, self.name)

    def run(self, command, name_addition=None, cmd_kwargs=None, _cmd="sbatch"):
        """
        command: a bash command that you want to run
        name_addition: if not specified, the sha1 of the command to run
                       appended to job name. if it is "date", the yyyy-mm-dd
                       date will be added to the job name.
        cmd_kwargs: dict of extra arguments to fill in command
                   (so command itself can be a template).
        _cmd: submit command (change to "bash" for testing).
        """
        if name_addition is None:
            name_addition = hashlib.sha1(command.encode("utf-8")).hexdigest()

        if self.date_in_name:
            name_addition += "-" + str(datetime.date.today())
        name_addition = name_addition.strip(" -")

        if cmd_kwargs is None:
            cmd_kwargs = {}

        n = self.name
        self.name = self.name.strip(" -")
        self.name += ("-" + name_addition.strip(" -"))

        tmpl = str(self).format(script=command)

        if "logs/" in tmpl and not os.path.exists("logs/"):
            os.makedirs("logs")

        with open(self._tmpfile(), "w") as sh:
            cmd_kwargs["script"] = command
            sh.write(tmpl.format(**cmd_kwargs))

        res = subprocess.check_output([_cmd, sh.name]).strip()
        print(res, file=sys.stderr)
        self.name = n
        if not res.startswith(b"Submitted batch"):
            return None
        job_id = int(res.split()[-1])
        return job_id

if __name__ == "__main__":
    import doctest
    doctest.testmod()

@@ -423,18 +423,27 @@
 ition=%22%22
+, tries=4
 )%0A
-
 %0A%22%22%22%0Afro
@@ -2180,24 +2180,33 @@
 cmd=%22sbatch%22
+, tries=1
 ):%0A        %22
@@ -2651,16 +2651,114 @@
 sting).%0A
+        tries: try to run a job either this many times or until the first%0A               success.%0A
         
@@ -3474,24 +3474,180 @@
 d_kwargs))%0A%0A
+        job_id = None%0A        for itry in range(1, tries + 1):%0A            if itry %3E 1:%0A                mid = %22--dependency=afternotok:%25d%22 %25 job_id%0A        
         res 
@@ -3678,16 +3678,21 @@
 t(%5B_cmd,
+ mid,
  sh.name
@@ -3698,24 +3698,117 @@
 e%5D).strip()%0A
+            else:%0A                res = subprocess.check_output(%5B_cmd, sh.name%5D).strip()%0A    
         prin
@@ -3831,32 +3831,36 @@
 stderr)%0A        
+   
+ 
 self.name = n%0A  
@@ -3857,16 +3857,20 @@
 ame = n%0A
+    
         
@@ -3920,24 +3920,28 @@
             
+    
 return None%0A
@@ -3944,27 +3944,29 @@
 one%0A        
+ 
-job
+   j
 _id = int(re
@@ -3980,16 +3980,72 @@
 ()%5B-1%5D)%0A
+            if itry == 1:%0A                job_id = j_id%0A
         

add option to run to try a script a number of times.