#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import subprocess, threading
import logging as loggers
import time
loggers.basicConfig(level=loggers.INFO)
logging = loggers.getLogger(__name__)


class ParallelExecutor(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None
        self.stdout_output = ""
        self.stderr_output = ""

    def run(self, timeout):
        logging.info(self.cmd)
        self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        self.stdout_output, self.stderr_output = self.process.communicate(timeout=timeout)
        logging.info("Process finished")

        self.process.terminate()
        time.sleep(1)
        try:
            self.process.kill()
        except OSError:
            pass
        return self.process.returncode

ERROR_KEYWORD = "Traceback (most recent call last)"

def test_experiment(path, timeout=60):
    path = os.path.join("experiments", path)
    executor = ParallelExecutor("python %s" % path)
    executor.run(timeout=timeout)
    if executor.stderr_output == None:
        logging.info("stderr is none")
        return
    if ERROR_KEYWORD in executor.stderr_output:
        logging.info("stdout:")
        logging.info(executor.stdout_output)
        logging.info("stderr:")
        logging.info(executor.stderr_output)
    assert ERROR_KEYWORD not in executor.stderr_output


if __name__ == '__main__':
    # MNIST
    test_experiment("mnist/mlp_dropout.py", timeout=30)
    test_experiment("mnist/deep_convolution.py", timeout=30)
    # LMs
    test_experiment("lm/baseline_rnnlm.py")
    # Highway networks
    test_experiment("highway_networks/mnist_highway.py")

@@ -49,16 +49,21 @@
 mport os
+, sys
 %0Aimport 
@@ -403,24 +403,50 @@
 , timeout):%0A
+        def target():%0A    
         logg
@@ -464,16 +464,20 @@
 lf.cmd)%0A
+    
         
@@ -578,32 +578,36 @@
 s.PIPE)%0A        
+    
 self.stdout_outp
@@ -660,67 +660,278 @@
 ate(
-timeout=timeout)%0A        logging.info(%22Process finished%22)%0A%0A
+)%0A            logging.info(%22Thread finished%22)%0A%0A        thread = threading.Thread(target=target)%0A        thread.daemon = True%0A        thread.start()%0A%0A        thread.join(timeout)%0A        if thread.is_alive():%0A            logging.info(%22Time out, terminating process%22)%0A    
     
@@ -967,16 +967,20 @@
         
+    
 time.sle
@@ -993,16 +993,20 @@
         
+    
 try:%0A   
@@ -1006,32 +1006,36 @@
 ry:%0A            
+   
+ 
 self.process.kil
@@ -1046,16 +1046,20 @@
         
+    
 except O
@@ -1066,16 +1066,121 @@
 SError:%0A
+                pass%0A            # thread.join()%0A        try:%0A            thread._stop()%0A        except:%0A
         
@@ -1890,25 +1890,25 @@
 y%22, timeout=
-3
+1
 0)%0A    test_
@@ -2089,12 +2089,28 @@
 ighway.py%22)%0A
+    sys.exit(0)%0A

Test experiments