#!/home/jake/.virtualenvs/giganews/bin/python
import sys
import time
import random

import logging
import yaml
import futures
from giganews import NewsGroup


log = logging.getLogger('giganews')


# archive_group()
# ________________________________________________________________________________________
def archive_group(group):
    # Hack to skip tricky groups for now.
    #if count > 11000 or 'binaries' in group:
    if 'binaries' in group:
        log.info('skipping {0} for now...'.format(group))
        return

    # TODO: fix this hack, which is used to skip groups only containig
    # a single level in their heirarchy. These groups conflict with items
    # from the historical usenet collection!
    if '.' not in group:
        return

    while True:
        try:
            user, password = ACCOUNTS.pop()
            break
        except IndexError:
            #log.debug('no accounts available, sleeping until one becomes available.')
            time.sleep(10)

    try:
        g = NewsGroup(group, user, password, logging_level='WARNING')

        # Check for new articles.
        count = int(g.last) - int(g.first)
        if count <= 0:
            log.info('no new articles found for {0}'.format(group))
            return

        g.archive_articles()

    finally:
        ACCOUNTS.append((user, password))


# init_accounts()
# ________________________________________________________________________________________
def init_accounts():
    global ACCOUNTS
    ACCOUNTS = yaml.load(open('/home/jake/.config/giganews.yml')).get('accounts', {}).items()


# ________________________________________________________________________________________
if __name__ == '__main__':
    init_accounts()

    news_list = [x.split()[0] for x in open('giganews_listfile.txt')]
    random.shuffle(news_list)

    try:
        # Concurrently archive all news for each newsgroup.
        with futures.ThreadPoolExecutor(max_workers=len(ACCOUNTS)) as e:
            future_to_group = {
                e.submit(archive_group, group): group for group in news_list
            }
            for future in futures.as_completed(future_to_group):
                group = future_to_group[future]
                result = group.result()
    except KeyboardInterrupt:
        sys.exit(1)

@@ -39,16 +39,84 @@
 /python%0A
+import gevent.monkey; gevent.monkey.patch_all()%0Aimport gevent.pool%0A%0A
 import s
@@ -176,23 +176,8 @@
 aml%0A
-import futures%0A
 from
@@ -202,16 +202,33 @@
 ewsGroup
+, GiganewsSession
 %0A%0A%0Alog =
@@ -820,254 +820,8 @@
 rn%0A%0A
-    while True:%0A        try:%0A            user, password = ACCOUNTS.pop()%0A            break%0A        except IndexError:%0A            #log.debug('no accounts available, sleeping until one becomes available.')%0A            time.sleep(10)%0A%0A    try:%0A    
     
@@ -841,30 +841,28 @@
 (group, 
-u
 se
-r, password
+ssion=sesh
 , loggin
@@ -874,27 +874,34 @@
 el='
-WARNING')%0A%0A   
+INFO', ia_sync=True)%0A%0A
     
- 
 # Ch
@@ -922,20 +922,16 @@
 ticles.%0A
-    
     coun
@@ -961,20 +961,16 @@
 .first)%0A
-    
     if c
@@ -984,28 +984,24 @@
  0:%0A        
- 
-   
 log.info('no
@@ -1048,28 +1048,24 @@
 p))%0A        
-    
 return%0A%0A    
@@ -1064,20 +1064,16 @@
 rn%0A%0A    
-  
-  
 g.archiv
@@ -1090,310 +1090,8 @@
 ()%0A%0A
-    finally:%0A        ACCOUNTS.append((user, password))%0A%0A%0A# init_accounts()%0A# ________________________________________________________________________________________%0Adef init_accounts():%0A    global ACCOUNTS%0A    ACCOUNTS = yaml.load(open('/home/jake/.config/giganews.yml')).get('accounts', %7B%7D).items()%0A%0A
 %0A# _
@@ -1213,21 +1213,46 @@
     
-init_accounts
+global sesh%0A    sesh = GiganewsSession
 ()%0A%0A
@@ -1355,473 +1355,143 @@
 st)%0A
-%0A
     
-try:%0A        # Concurrently archive all news for each newsgroup.%0A        with futures.ThreadPoolExecutor(max_workers=len(ACCOUNTS)) as e:%0A            future_to_group = %7B%0A                e.submit(archive_group, group): group for group in news_list%0A            %7D%0A            for future in futures.as_completed(future_to_group):%0A                group = future_to_group%5Bfuture%5D%0A                result = group.result()%0A    except KeyboardInterrupt:%0A        sys.exit(1
+pool = gevent.pool.Pool(16)%0A    pool.map(archive_group, news_list)%0A    #for g in news_list:%0A    #    print g%0A    #    archive_group(g
 )%0A

use gevent instead of concurrent.futures.