"""
License:
   Copyright 2010 Anthony Kolesov

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

import threading, time, logging, uuid, xmlrpclib
from xmlrpclib import ServerProxy
import kts46.utils
from kts46.serverApi import RPCServerException
from kts46.CouchDBStorage import CouchDBStorage
from kts46.simulationServer import SimulationServer
from kts46.statisticsServer import StatisticsServer

def _notificationThreadImplementation(worker):
    while True:
        worker.enableNotificationEvent.wait()
        time.sleep(worker.notificationSleepTimeout)

        # Acquire lock here so worker will not remove it after condition test.
        worker.lastUpdateLock.acquire()
        # Event may have been disabled while we were waiting, so check here again.
        if worker.enableNotificationEvent.is_set():
            worker.log.info('[%s] Sending notification to server...', threading.currentThread().name)
            lu = worker.server.reportStatus(worker.workerId, 'working', worker.lastUpdate)
            worker.lastUpdate = lu
        worker.lastUpdateLock.release()


class WorkerException(Exception): pass

class Worker:

    def __init__(self, cfg, workerId=None):
        self.cfg = cfg
        self.log = logging.getLogger(cfg.get('loggers', 'Worker'))

        # Setup worker id. It must be unique in a workers network.
        if workerId is not None:
            self.workerId = workerId
        elif cfg.has_option('worker', 'id'):
            self.workerId = cfg.get('worker', 'id')
        else:
            self.workerId = str(uuid.uuid4())

        self.enableNotificationEvent = threading.Event()
        self.lastUpdateLock = threading.Lock()
        # Take default timeout from scheduler configuration.
        self.notificationSleepTimeout = cfg.getint('scheduler', 'notifyInterval')

        # Create server proxy.
        self.server = kts46.utils.getRPCServerProxy(cfg)
        self.startNotificationThread()

        # Create db storage.
        self.storage = CouchDBStorage(self.cfg.get('couchdb', 'dbaddress'))


    def run(self):
        "Runs a worker loop."

        # Start run loop.
        while True:
            # Try to get a task.
            task = self.server.getJob(self.workerId)

            # Sleep if there is nothing to do.
            # task is a AutoProxy, not None. So we coudn't check for `is None`. May be there
            # is a better way than comparing strings but that works.
            if str(task) == "None":
                sleepTime = self.cfg.getfloat('worker', 'checkInterval')
                self.log.debug('Worker has nothing to do. Sleeping for %f s.', sleepTime)
                time.sleep(sleepTime) # Wait some time for new job.
                continue

            # There is a task.
            projectName = task.get('project')
            jobName = task.get('job')
            # Report status isn't enabled at this time, so no need to lock.
            self.lastUpdate = task.get('lastUpdate')

            self.notificationSleepTimeout = task.get('timeout')
            self.enableNotificationEvent.set() # Start notifying scheduler about our state.
            job = self.getJob(projectName, jobName)

            if task.get('type') == 'simulation':
                self.log.info('Starting simulation task: {0}.{1}.'.format(projectName, jobName))
                simServer = SimulationServer()
                simServer.runSimulationJob(job)
            elif task.get('type') == 'statistics':
                self.log.info('Starting statistics task: {0}.{1}.'.format(projectName, jobName))
                stServer = StatisticsServer(self.cfg)
                stServer.calculate(projectName, job)

            # Notify server.
            # Lock here so if condition in sync thread will be correct.
            self.lastUpdateLock.acquire()
            self.enableNotificationEvent.clear() # Stop notifying scheduler.
            self.server.reportStatus(self.workerId, 'finished', self.lastUpdate)
            self.lastUpdateLock.release()


    def startNotificationThread(self):
        "Starts thread to notify scheduler about worker availability and returns Thread object."
        t = threading.Thread(target=_notificationThreadImplementation, kwargs={'worker':self})
        t.daemon = True
        t.start()
        return t


    def getJob(self, projectName, jobName):
        if projectName not in self.storage:
            raise WorkerException("Project '{0}' doesn't exist.".format(projectName))
        project = self.storage[projectName]

        if jobName not in project:
            msg = "Job with name '{0}' doesn't exist in project '{1}'."
            raise WorkerException(msg.format(jobName, projectName))
        return project[jobName]

@@ -23,16 +23,21 @@
 ght 2010
+-2011
  Anthony
@@ -45,16 +45,16 @@
 Kolesov%0A
-
 %0A   Lice
@@ -685,16 +685,60 @@
 erProxy%0A
+from socket import error as SocketException%0A
 import k
@@ -1462,24 +1462,45 @@
 ead().name)%0A
+            try:%0A    
             
@@ -1570,32 +1570,324 @@
 ker.lastUpdate)%0A
+            except SocketException, msg:%0A                # Continue to work even if connection failed. Hope that next%0A                # time will be successful.%0A                worker.log.warning(%22Connection to RPC server failed. Couldn't notify server about task status. Continue to work.%22)%0A
             work
@@ -3036,32 +3036,53 @@
  to get a task.%0A
+            try:%0A    
             task
@@ -3117,16 +3117,168 @@
 orkerId)
+%0A            except SocketException, msg:%0A                self.log.error(%22Couldn't connect to RPC server. Message: %25s%22, msg)%0A                task = None
 %0A%0A      
@@ -4960,75 +4960,422 @@
     
-self.server.reportStatus(self.workerId, 'finished', self.lastUpdate
+finishedSent = False%0A            while not finishedSent:%0A                try:%0A                    self.server.reportStatus(self.workerId, 'finished', self.lastUpdate)%0A                    finishedSent = True%0A                except SocketException, msg:%0A                    self.log.error(%22Connection to RPC server failed. Waiting for it.%22)%0A                    time.sleep(self.cfg.getfloat('worker', 'checkInterval')
 )%0A  

Add to worker some checks for case if RPC server is unavailable. Now it will not fail, but continue to wait for server come back.