from importlib import reload

import pandas as pd
import numpy as np
from textblob import TextBlob
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from scipy.spatial.distance import pdist
from sklearn.preprocessing import MultiLabelBinarizer

# Add general functions to the project
from os import path
import sys
sys.path.append(path.abspath('../methods'))

import embedding
import vis

# DATA Loading
raw = np.load("../datasets/full.pkl")

# Remove Unclear from groups/clusters
# raw["Clusters"] = data.apply(
#     lambda row: row["Clusters"]
#     .replace(";Unclear", "")
#     .replace(";;", ";"), axis=1
#     )

# train/test split
test = raw[raw["DOI"].str.contains("2013|2012")]  # len = 197
train = raw.drop(test.index)  # len = 1280


labels = ["Clusters"]

# data

# y
enc = MultiLabelBinarizer()
enc.fit([cluster.split(";") for cluster in train[labels]["Clusters"].tolist()])

y_train = train[labels].apply(lambda row: enc.transform([row["Clusters"].split(";")])[0], axis=1).values

# prepare for visualization

# Analysis
# Similarity
metric = "jaccard"  # cosine, jaccard, emd, cm
 distance = embedding.similarity_matrix(y_train, metric, as_distance=True)

# vis.simMatrix(distance)
vis.scree_plot(distance, vecs, nonlinear=False, uselda=False, usenmf=False)
vis.graph(embedding.graph_from_dist(distance, 0.60), labels)
# vis.cluster_heatmap(
#     np.array(vecs),
#     metric=metric,
#     mode="intersection",
#     order=True)
vis.scatter(vecs, labels)
vis.scatter_tsne(vecs, labels, 0.1)


# Plot local graph for a document
local_neigh = np.argwhere(distance[1, :] < 0.80).flatten()
local_dist = distance[np.ix_(local_neigh, local_neigh)]
local_labels = np.array(labels)[local_neigh]

vis.graph(
    embedding.graph_from_dist(local_dist, local_dist.mean()), local_labels
    )

# Compute jaccard distance
jaccard_distances = np.zeros([n, n])
for i in range(0, n):
    for j in range(0, n):
        a = set(data.iloc[i]["Clusters"].split(";"))
        b = set(data.iloc[j]["Clusters"].split(";"))

        jd = len(a.intersection(b))/len(a.union(b))

        jaccard_distances[i][j] = jd

@@ -521,16 +521,49 @@
 ll.pkl%22)
+%0Araw = raw.reset_index(drop=True)
 %0A%0A# Remo
@@ -594,18 +594,16 @@
 lusters%0A
-# 
 raw%5B%22Clu
@@ -616,20 +616,19 @@
 %5D = 
-data
+raw
 .apply(%0A
 #   
@@ -623,18 +623,16 @@
 .apply(%0A
-# 
     lamb
@@ -655,22 +655,20 @@
 sters%22%5D%0A
-#
     
- 
 .replace
@@ -669,17 +669,16 @@
 eplace(%22
-;
 Unclear%22
@@ -683,18 +683,16 @@
 r%22, %22%22)%0A
-# 
     .rep
@@ -719,35 +719,369 @@
 s=1%0A
-#     )%0A%0A# train/test split
+    )%0A%0A%0A# METADATA%0Ameta = raw.drop(%5B%22Fulltext%22%5D, axis=1)%0Ameta%5B'type'%5D = meta.apply(lambda row: 'new' if any(x in row%5B%22DOI%22%5D for x in %5B%222013%22, %222012%22%5D) else 'old', axis=1)%0Ameta.to_json(%22meta.json%22, orient=%22index%22)%0A%0Afulltext = raw%5B%22Fulltext%22%5D%0Afulltext.to_json(%22fulltext.json%22, orient=%22index%22)%0A%0A# EMBEDDINGS%0A%0A%0A# CLASSIFICATION%0A# train/test split for classification
 %0Ates
@@ -1187,38 +1187,11 @@
 80%0A%0A
-%0Alabels = %5B%22Clusters%22%5D%0A%0A# data
+raw
 %0A%0A# 
@@ -1273,16 +1273,8 @@
 ain%5B
-labels%5D%5B
 %22Clu
@@ -1309,22 +1309,26 @@
 = train%5B
-labels
+%22Clusters%22
 %5D.apply(
@@ -1503,17 +1503,16 @@
 emd, cm%0A
- 
 distance
@@ -1614,907 +1614,60 @@
 s.sc
-ree_plot(distance, vecs, nonlinear=False, uselda=False, usenmf=False)%0Avis.graph(embedding.graph_from_dist(distance, 0.60), labels)%0A# vis.cluster_heatmap(%0A#     np.array(vecs),%0A#     metric=metric,%0A#     mode=%22intersection%22,%0A#     order=True)%0Avis.scatter(vecs, labels)%0Avis.scatter_tsne(vecs, labels, 0.1)%0A%0A%0A# Plot local graph for a document%0Alocal_neigh = np.argwhere(distance%5B1, :%5D %3C 0.80).flatten()%0Alocal_dist = distance%5Bnp.ix_(local_neigh, local_neigh)%5D%0Alocal_labels = np.array(labels)%5Blocal_neigh%5D%0A%0Avis.graph(%0A    embedding.graph_from_dist(local_dist, local_dist.mean()), local_labels%0A    )%0A%0A# Compute jaccard distance%0Ajaccard_distances = np.zeros(%5Bn, n%5D)%0Afor i in range(0, n):%0A    for j in range(0, n):%0A        a = set(data.iloc%5Bi%5D%5B%22Clusters%22%5D.split(%22;%22))%0A        b = set(data.iloc%5Bj%5D%5B%22Clusters%22%5D.split(%22;%22))%0A%0A        jd = len(a.intersection(b))/len(a.union(b))%0A%0A        jaccard_distances%5Bi%5D%5Bj%5D = jd
+atter(vecs, labels)%0Avis.scatter_tsne(vecs, labels, 0.1)
 %0A

save meta informations