@@ -0,0 +1,2274 @@
+import logging%0Afrom threading import Thread%0Atry:%0A    # Python 2.x%0A    import Queue as queue%0Aexcept ImportError:%0A    # Python 3%0A    import queue%0A%0Aclass ErrorObject(object):%0A    def __init__(self, *args, **kwargs):%0A        self.error_occurred = False%0A%0Adef worker_thread(q, err, function, *args, **kwargs):%0A    while True:%0A        try:%0A            item = q.get_nowait()%0A        except queue.Empty:%0A            break%0A        except Exception as e:%0A            # No need to do any locking here%0A            err.error_occurred = True%0A            logging.exception(%22Error in worker_thread%22)%0A            break%0A%0A        try:%0A            function(item, *args, **kwargs)%0A        except Exception as e:%0A            err.error_occurred = True%0A            logging.exception(%22Error in %25s%22 %25 function.__name__)%0A%0A        q.task_done()%0A%0Adef process_fast(items, function, num_threads=4, item_goal=None, max_threads=50,%0A    args=None, kwargs=None):%0A    %22%22%22%0A    Spawns a number of threads which will process the given items.%0A    items - the list of items to process%0A    function - the function to call on each item; should accept the item as the%0A      first argument, and any other args or kwargs sent here%0A    num_threads - number of threads to use if item_goal is None%0A    item_goal - if specified, the number of threads will be chosen so that each%0A      thread should process about this many items%0A    max_threads - used as a limit to the number of threads if item_goal is%0A      specified%0A%0A    This function does not return until all items are processed.%0A    Raises RuntimeError if at least one thread raised an exception. Thread%0A      exceptions are logged using logging.exception.%0A    %22%22%22%0A    if item_goal:%0A        num_threads = (len(items) / item_goal) + 1%0A        num_threads = min(num_threads, max_threads)%0A%0A    q = queue.Queue()%0A    for item in items:%0A        q.put(item)%0A%0A    err = ErrorObject()%0A    passed_args = (q, err, function) + (args or ())%0A    kwargs = kwargs or %7B%7D%0A    for i in range(num_threads):%0A        t = Thread(target=worker_thread, args=passed_args, kwargs=kwargs)%0A        t.start()%0A%0A    # Wait until the queue is empty%0A    q.join()%0A    if err.error_occurred:%0A        raise RuntimeError(%22Error occurred calling process_fast. Check %22%0A            %22couchlog for details.%22)%0A%0A

Add multithreading util