import logging
import glob
from multiprocessing import Pool

from CPTCorpus import CPTCorpus
from CPT_Gibbs import GibbsSampler


def run_sampler(corpus, nTopics, nIter, beta, out_dir):
    alpha = 50.0/nTopics
    logger.info('running Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
                'beta: {})'.format(nTopics, nIter, alpha, beta))
    sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
                           alpha=alpha, beta=beta, beta_o=beta,
                           out_dir=out_dir.format(nTopics))
    sampler._initialize()
    sampler.run()


logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

logger.setLevel(logging.DEBUG)
#logger.setLevel(logging.INFO)

files = glob.glob('/home/jvdzwaan/data/dilipad/20112012/gov_opp/*')

out_dir = '/home/jvdzwaan/data/dilipad/res_20112012/{}'

corpus = CPTCorpus(files, testSplit=20)
corpus.filter_dictionaries(minFreq=5, removeTopTF=100, removeTopDF=100)
corpus.save_dictionaries(directory=out_dir.format(''))
corpus.save(out_dir.format('corpus.json'))

#corpus = CPTCorpus.load(out_dir.format('corpus.json'),
#                        topicDict=out_dir.format('topicDict.dict'),
#                        opinionDict=out_dir.format('opinionDict.dict'))

nIter = 200
beta = 0.02

nTopics = range(20, 201, 20)
logger.info('running Gibbs sampler for {} configurations'.format(len(nTopics)))

pool = Pool(processes=3)
results = [pool.apply_async(run_sampler, args=(corpus, n, nIter, beta,
                                               out_dir))
           for n in nTopics]
pool.close()
pool.join()

@@ -52,16 +52,44 @@
 ort Pool
+%0Aimport argparse%0Aimport json
 %0A%0Afrom C
@@ -196,16 +196,24 @@
 r, beta,
+ beta_o,
  out_dir
@@ -339,16 +339,28 @@
   'beta:
+ %7B%7D, beta_o:
  %7B%7D)'.fo
@@ -391,16 +391,71 @@
 ha, beta
+,%0A                                               beta_o
 ))%0A    s
@@ -572,24 +572,26 @@
  beta_o=beta
+_o
 ,%0A          
@@ -758,20 +758,21 @@
 logging.
-INFO
+DEBUG
 )%0Alogger
@@ -807,197 +807,908 @@
 _)%0A%0A
-logger.setLevel(logging.DEBUG)%0A#logger.setLevel(logging.INFO)%0A%0Afiles = glob.glob('/home/jvdzwaan/data/dilipad/20112012/gov_opp/*')%0A%0Aout_dir = '/home/jvdzwaan/data/dilipad/res_20112012/%7B%7D'%0A%0A
+parser = argparse.ArgumentParser()%0Aparser.add_argument('json', help='json file containing experiment '%0A                    'configuration.')%0Aargs = parser.parse_args()%0A%0Awith open(args.json) as f:%0A    config = json.load(f)%0A%0Alogger.debug('configuration of experiment: ')%0Aparams = %5B'%7B%7D: %7B%7D'.format(p, v) for p, v in config.iteritems()%5D%0Afor p in params:%0A    logger.debug(p)%0A%0Afiles = glob.glob(config.get('input_data'))%0A%0Aout_dir = config.get('out_dir', '/%7B%7D')%0AtestSplit = config.get('testSplit', 20)%0AminFreq = config.get('minFreq', 5)%0AremoveTopTF = config.get('removeTopTF', 100)%0AremoveTopDF = config.get('removeTopDF', 100)%0AnIter = config.get('nIter', 200)%0Abeta = config.get('beta', 0.02)%0Abeta_o = config.get('beta_o', 0.02)%0AnTopics = config.get('nTopics', range(20, 201, 20))%0AnProcesses = config.get('nProcesses', None)%0AloadDictionaries = config.get('loadDictionaries', 0)%0A%0Aif not loadDictionaries:%0A    
 corp
@@ -1739,20 +1739,31 @@
 stSplit=
-20)%0A
+testSplit)%0A    
 corpus.f
@@ -1789,17 +1789,23 @@
 minFreq=
-5
+minFreq
 , remove
@@ -1814,30 +1814,81 @@
 pTF=
-100, removeTopDF=100)%0A
+removeTopTF,%0A                               removeTopDF=removeTopDF)%0A    
 corp
@@ -1938,16 +1938,20 @@
 at(''))%0A
+    
 corpus.s
@@ -1985,18 +1985,26 @@
 json'))%0A
-%0A#
+else:%0A    
 corpus =
@@ -2046,25 +2046,28 @@
 pus.json'),%0A
-#
+    
             
@@ -2126,9 +2126,12 @@
 '),%0A
-#
+    
     
@@ -2203,62 +2203,8 @@
 ))%0A%0A
-nIter = 200%0Abeta = 0.02%0A%0AnTopics = range(20, 201, 20)%0A
 logg
@@ -2306,9 +2306,18 @@
 ses=
-3
+nProcesses
 )%0Are
@@ -2380,24 +2380,32 @@
 nIter, beta,
+ beta_o,
 %0A           

Make experiments with number of topics configurable