"""
Output some statistics for the real graph datasets.  

"""
import os
import sys
import logging
import numpy
import itertools 
from exp.clusterexp.CitationIterGenerator import CitationIterGenerator 
from exp.clusterexp.HIVIterGenerator import HIVIterGenerator 
from apgl.graph import GraphStatistics 
from apgl.graph import SparseGraph, GraphUtils  
from apgl.util.PathDefaults import PathDefaults
from exp.clusterexp.BemolData import BemolData
import matplotlib.pyplot as plt 
import scipy.sparse.linalg 

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

plotHIV = False 
plotCitation = False
plotBemol = True 

if plotHIV: 
    generator = HIVIterGenerator()
    iterator = generator.getIterator()
    
    resultsDir = PathDefaults.getOutputDir() + "cluster/HIV/Stats/"
if plotCitation: 
    maxGraphSize = 3000 
    generator = CitationIterGenerator(maxGraphSize=maxGraphSize)
    iterator = generator.getIterator()
    
    resultsDir = PathDefaults.getOutputDir() + "cluster/Citation/Stats/"
if plotBemol: 
    dataDir = PathDefaults.getDataDir() + "cluster/"
    nbUser = 20000 # set to 'None' to have all users
    nbPurchasesPerIt = 50 # set to 'None' to take all the purchases
                                          # per date
    startingIteration = 0
    endingIteration = None # set to 'None' to have all iterations
    stepSize = 10    
    
    iterator = itertools.islice(BemolData.getGraphIterator(dataDir, nbUser, nbPurchasesPerIt), startingIteration, endingIteration, stepSize)
    resultsDir = PathDefaults.getOutputDir() + "cluster/Bemol/Stats/"
    #Appears to be 10 components 

subgraphIndicesList = []
for W in iterator: 
    logging.debug("Graph size " + str(W.shape[0]))
    subgraphIndicesList.append(range(W.shape[0])) 

#Try to find number of clusters at end of sequence by looking at eigengap 
k = 2000
L = GraphUtils.normalisedLaplacianSym(W)

logging.debug("Computing eigenvalues")
omega, Q = scipy.sparse.linalg.eigsh(L, min(k, L.shape[0]-1), which="SM", ncv = min(20*k, L.shape[0]))

omegaDiff = numpy.diff(omega)

plotInd = 0 
plt.figure(plotInd)
plt.plot(numpy.arange(omega.shape[0]), omega)
plt.xlabel("Eigenvalue index")
plt.ylabel("Eigenvalue")
plt.savefig(resultsDir + "Spectrum.eps")
plotInd+=1 

plt.figure(plotInd)
plt.plot(numpy.arange(omegaDiff.shape[0]), omegaDiff)
plt.xlabel("Eigenvalue index")
plt.ylabel("Eigenvalue diff")
plt.savefig(resultsDir + "SpectrumDiff.eps")
plotInd +=1 

#No obvious number of clusters and there are many edges 
graph = SparseGraph(W.shape[0], W=W)

logging.debug("Computing graph statistics")
graphStats = GraphStatistics()
statsMatrix = graphStats.sequenceScalarStats(graph, subgraphIndicesList, slowStats=False)

plt.figure(plotInd)
plt.plot(numpy.arange(statsMatrix.shape[0]), statsMatrix[:, graphStats.numVerticesIndex])
plt.xlabel("Graph index")
plt.ylabel("Num vertices")
plt.savefig(resultsDir + "Vertices.eps")
plotInd+=1 


plt.figure(plotInd)
plt.plot(numpy.arange(statsMatrix.shape[0]), statsMatrix[:, graphStats.numEdgesIndex])
plt.xlabel("Graph index")
plt.ylabel("Num edges")
plt.savefig(resultsDir + "Edges.eps")
plotInd+=1 

plt.figure(plotInd)
plt.plot(numpy.arange(statsMatrix.shape[0]), statsMatrix[:, graphStats.maxComponentSizeIndex])
plt.xlabel("Graph index")
plt.ylabel("Max component size")
plotInd+=1 

plt.figure(plotInd)
plt.plot(numpy.arange(statsMatrix.shape[0]), statsMatrix[:, graphStats.numComponentsIndex])
plt.xlabel("Graph index")
plt.ylabel("Num components")
plt.savefig(resultsDir + "Components.eps")
plotInd+=1 

plt.show()

@@ -1761,16 +1761,47 @@
 %5B0%5D)) %0A%0A
+plotInd = 0 %0AfindEigs = False%0A%0A
 #Try to 
@@ -1871,17 +1871,36 @@
 ap %0A
-k = 2000%0A
+if findEigs: %0A    k = 2%0A    
 L = 
@@ -1936,17 +1936,25 @@
 nSym(W)%0A
-%0A
+    %0A    
 logging.
@@ -1984,16 +1984,20 @@
 alues%22)%0A
+    
 omega, Q
@@ -2091,17 +2091,25 @@
 pe%5B0%5D))%0A
-%0A
+    %0A    
 omegaDif
@@ -2130,30 +2130,30 @@
 (omega)%0A
-%0AplotInd = 0 %0A
+    %0A    %0A    
 plt.figu
@@ -2156,32 +2156,36 @@
 figure(plotInd)%0A
+    
 plt.plot(numpy.a
@@ -2214,16 +2214,20 @@
  omega)%0A
+    
 plt.xlab
@@ -2241,32 +2241,36 @@
 envalue index%22)%0A
+    
 plt.ylabel(%22Eige
@@ -2278,16 +2278,20 @@
 value%22)%0A
+    
 plt.save
@@ -2319,24 +2319,28 @@
 ctrum.eps%22)%0A
+    
 plotInd+=1 %0A
@@ -2331,33 +2331,41 @@
     plotInd+=1 %0A
-%0A
+    %0A    
 plt.figure(plotI
@@ -2360,32 +2360,36 @@
 figure(plotInd)%0A
+    
 plt.plot(numpy.a
@@ -2426,16 +2426,20 @@
 gaDiff)%0A
+    
 plt.xlab
@@ -2453,32 +2453,36 @@
 envalue index%22)%0A
+    
 plt.ylabel(%22Eige
@@ -2495,16 +2495,20 @@
  diff%22)%0A
+    
 plt.save
@@ -2540,24 +2540,28 @@
 mDiff.eps%22)%0A
+    
 plotInd +=1 
@@ -2823,16 +2823,68 @@
 False)%0A%0A
+numpy.save(resultsDir + %22GraphStats%22, statsMatrix)%0A%0A
 plt.figu

Save stats and exclude eigenvalues.