import pycurl

from grab.util.py3k_support import *

class MulticurlTransport(object):
    def __init__(self, thread_number):
        self.thread_number = thread_number
        self.multi = pycurl.CurlMulti()
        self.multi.handles = []
        self.freelist = []
        self.registry = {}

        # Create curl instances
        for x in xrange(self.thread_number):
            curl = pycurl.Curl()
            self.freelist.append(curl)
            self.multi.handles.append(curl)

    def ready_for_task(self):
        return len(self.freelist)

    def active_task_number(self):
        return self.thread_number - len(self.freelist)

    def process_task(self, task, grab, grab_config_backup):
        curl = self.freelist.pop()
        curl.task = task
        self.registry[id(curl)] = {
            'grab': grab,
            'grab_config_backup': grab_config_backup,
            'task': task,
        }
        grab.transport.curl = curl
        grab.prepare_request()
        grab.log_request()

        # Add configured curl instance to multi-curl processor
        self.multi.add_handle(curl)

    def process_handlers(self):
        # http://curl.haxx.se/libcurl/c/curl_multi_perform.html
        while True:
            status, active_objects = self.multi.perform()
            if status != pycurl.E_CALL_MULTI_PERFORM:
                break

    def iterate_results(self):
        while True:
            queued_messages, ok_list, fail_list = self.multi.info_read()

            results = []
            for curl in ok_list:
                results.append((True, curl, None, None))
            for curl, ecode, emsg in fail_list:
                # CURLE_WRITE_ERROR (23)
                # An error occurred when writing received data to a local file, or
                # an error was returned to libcurl from a write callback.
                # This exception should be ignored if _callback_interrupted flag
                # is enabled (this happens when nohead or nobody options enabeld)
                #
                # Also this error is raised when curl receives KeyboardInterrupt
                # while it is processing some callback function
                # (WRITEFUNCTION, HEADERFUNCTIO, etc)
                if ecode == 23:
                    if getattr(curl, '_callback_interrupted', None) == True:
                        curl._callback_interrupted = False
                        ecode = None
                        emsge = None
                        results.append((True, curl, None, None))
                    else:
                        results.append((False, curl, ecode, emsg))
                else:
                    results.append((False, curl, ecode, emsg))

            for ok, curl, ecode, emsg in results:
                # FORMAT: {ok, grab, grab_config_backup, task, emsg}

                curl_id = id(curl)
                task = self.registry[curl_id]['task']
                grab = self.registry[curl_id]['grab']
                grab_config_backup = self.registry[curl_id]['grab_config_backup']

                grab.process_request_result()

                # Free resources
                del self.registry[curl_id]

                yield {'ok': ok, 'emsg': emsg, 'grab': grab,
                       'grab_config_backup': grab_config_backup, 'task': task}

                self.multi.remove_handle(curl)
                self.freelist.append(curl)

            if not queued_messages:
                break

    def select(self, timeout=0.01):
        return self.multi.select(timeout)

@@ -287,16 +287,51 @@
 try = %7B%7D
+%0A        self.connection_count = %7B%7D
 %0A%0A      
@@ -434,16 +434,64 @@
 .Curl()%0A
+            self.connection_count%5Bid(curl)%5D = 0%0A
         
@@ -525,32 +525,33 @@
 rl)%0A            
+#
 self.multi.handl
@@ -714,32 +714,422 @@
 self.freelist)%0A%0A
+    def process_connection_count(self, curl):%0A        curl_id = id(curl)%0A        self.connection_count%5Bcurl_id%5D += 1%0A        if self.connection_count%5Bcurl_id%5D %3E 100:%0A            del self.connection_count%5Bcurl_id%5D%0A            del curl%0A            new_curl = pycurl.Curl()%0A            self.connection_count%5Bid(new_curl)%5D = 1%0A            return new_curl%0A        else:%0A            return curl%0A%0A
     def process_
@@ -1196,47 +1196,54 @@
 elf.
-freelist.pop()%0A        curl.task = task
+process_connection_count(self.freelist.pop())%0A
 %0A   
@@ -1422,24 +1422,41 @@
 curl = curl%0A
+        try:%0A    
         grab
@@ -1478,24 +1478,28 @@
 t()%0A        
+    
 grab.log_req
@@ -1501,25 +1501,307 @@
 g_request()%0A
-%0A
+        except Exception, ex:%0A            # If some error occured while processing the request arguments%0A            # then we should put curl object back to free list%0A            del self.registry%5Bid(curl)%5D%0A            self.freelist.append(curl)%0A            raise%0A        else:%0A    
         # Ad
@@ -1851,16 +1851,20 @@
 ocessor%0A
+    
         
@@ -2000,32 +2000,65 @@
     while True:%0A
+            #print '%5Binside PH%5D'%0A
             stat
@@ -3996,16 +3996,314 @@
 curl_id%5D
+%0A                grab.transport.curl = None%0A%0A                #if emsg and 'Operation timed out after' in emsg:%0A                    #num =  int(emsg.split('Operation timed out after')%5B1%5D.strip().split(' ')%5B0%5D)%0A                    #if num %3E 20000:%0A                        #import pdb; pdb.set_trace()
 %0A%0A      

Enable refreshing multicurl handlers with new ones after they have processed a number of connections