# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.15.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import spacy
from spacy.tokens import Span
from spacy.matcher import PhraseMatcher

nlp = spacy.load("pt_core_news_sm")
doc = nlp("Eu gosto muito de jogar a bola.")

print(doc.text)
animals = ["Golden Retriever", "cat", "turtle", "Rattus norvegicus"]
matcher = PhraseMatcher(nlp.vocab)


# -

def animal_component(doc):
    animals = ["Golden Retriever", "cat", "turtle", "Rattus norvegicus"]
    matcher = PhraseMatcher(nlp.vocab)
    
    matches = matcher(doc)
    span = [Span(doc, start, end, label="ANIMAL") for match_id, start, end in matches]
    doc.ents = span
    return doc


# +
nlp.add_pipe(animal_component, after=False)

time_spent = []

# +
from time import time

def timer(func):
    def f(*args, **kwargs):
        global time_spent
        before = time()
        rv = func(*args, **kwargs)
        after = time()
        time_spent[func.__name__] = format(after - before, '.2f')# round(int(after - before), 3)
        return rv
    return f


# +
@timer
def get_entities(doc):
    return [(ent.text, ent.label_) for ent in doc.ents]

def get_verbs(doc):
    return [(token.text, token.pos_) for token in doc if token.pos_ == "VERB" or token.pos_ == "AUX"]

@timer
def get_countries(doc):
    return [(ent.text, ent.label_) for ent in doc.ents if ent.label_ == 'GPE']


def get_persons(doc):
    return [(ent.text, ent.label_) for ent in doc.ents]



# +
TEXT = """
Eu gosto muito de jogar a bola com o meu amigo Manuel e o João.
"""
doc = nlp(TEXT)

print('PERSONS   :',  get_persons(doc))
print('VERBS   :',  get_verbs(doc))
print(time_spent)

# +
from transformers import pipeline

classifier = pipeline("sentiment-analysis")

# +
results = classifier(["We are very happy to show you the 🤗 Transformers library", "We hope you don't hate it."])

for result in results:
    print(f"label: {result['label']}, with score: {round(result['score'], 4)}")

# +
import csv

p = '../dataset/datasets_73978_166769_fb_sentiment.csv'
tweets = []
with open(p, newline='') as f:
    spamreader = csv.DictReader(f)
    for row in spamreader:
        tweets.append(row["FBPost"])
        
tweets[0]

# +
import pandas as pd
import collections
from collections import Counter
import numpy as np

review_subset = pd.read_csv('/home/yanick/KRAKEN/PIP/NLP/data/Yelp Dataset/yelp.csv')

start = 10
end = 20

print(len(review_subset)/10)

by_rating = [] #collections.defaultdict(list)

for _, row in review_subset[start:end].iterrows():
    by_rating.append({
        "review_id": row.review_id,
        "text": row.text
    })

by_rating

# +


results = classifier(" really I cant believe this place has received such high reviews from people\n\nmy lady and i walked in and were greeted rather rudely by a pretentious bitch at the front with a monotone name please instead of a warm friendly french welcome to this rather charming looking place we didnt have a reservation which from the looks of how dead empty the place was didnt seem like a problem until the hostess whisked through her reservation list and explained shed try to fit us in it was 6pm the place had one other couple dining and we were informed that the next party was arriving at 730 we werent really looking to hang out it would have be awfully lovely if yelpers would have informed of the need to MAKE RESERVATIONS before going in here because if you dont youre automatically a piece of shit according to the staff sorry i dont plan very much ahead next time i will and it wont be here\n\nanyways after debating for a minute or two with herself the hostess decided we could squeeze in and sat us right on top of the other couple inside the restaurant she informed us that we could cork our own wine for their low low price of 9 fucking dollars in words only office space could adequately describecoup des tartes what is it exactly that you do \n\nthe food was good nothing spectacular for 25 a plate we got a salad with apples and nuts and bleu cheese which was served warm now this is the first warm salad ive had all of my life and by no means am i some sort of wordly food expert but i believe salad should be cold and crisp not soggy and warm yuck the cordon bleu was tasty and filling however the wobbly table was an extreme annoyance when cutting through the chicken but it was solved with no thanks to the staff by placing a jackinthebox gift card under the leg i will admit the mashed potatoes are some of the creamiest and best ive ever had my lady got the pork tenderloin which had a strange pepperysweet pairing which i didnt particularly care for we were broke for desert so unfortunately we didnt get to try any tarts you would think for a place that promotes itself as being so poise would get some fancier menus other than a 15 piece of cardstock copied in bulk at kinkos\n\nthis place was rather disappointing i honestly thought it would be way more comfy and welcoming inside but it really wasnt it presents itself as being so high class and french and ohlala wee wee but when it comes down to it its still located in a tiny shack of a house at 16th street and highland maybe if it were located in the biltmore id give it a bit more cred ")
print("RESULT:  ", results)

# +
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")

# +
from transformers import pipeline
from progress.bar import Bar

context = r"""
Extractive Question Answering is the task of extracting an answer from a text given a question. An example of a
question answering dataset is the SQuAD dataset, which is entirely based on that task. If you would like to fine-tune
a model on a SQuAD task, you may leverage the examples/question-answering/run_squad.py script.
"""

bar = Bar('Processing', max=20)

loading = True
while loading:
    bar.next()
    nlp = pipeline("question-answering")
    loading = False
bar.finis
print(type(nlp))
result = nlp(question="What is extractive question answering?", context=context)
print(result)
# -

nlp = pipeline("ner")

# +
from transformers import pipeline

summarizer = pipeline("summarization")

# +
ARTICLE = """ New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York.
A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband.
Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other.
In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage.
Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the
2010 marriage license application, according to court documents.
Prosecutors said the marriages were part of an immigration scam.
On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further.
After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective
Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002.
All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say.
Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages.
Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted.
The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s
Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali.
Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force.
If convicted, Barrientos faces up to four years in prison.  Her next court appearance is scheduled for May 18.
"""

#s = summarizer(ARTICLE, max_length=118, min_length=30, do_sample=False)
#print(s[0]["summary_text"])

# +
ARTICLE = """
In order to analyze what has been done in Cape Verde with regard to Intellectual Property 
and also Digital Signature, we interviewed the competent entities related to each particular subject
and conducted interviews and applied questionnaires that we sent through electronic mails 
and social networks to our artists and authors both in Cape Verde and some who are living
abroad. With the data collected it was possible to verify that the protection of Intellectual
Property is still in a phase of maturation in our archipelago and having been given more
attention in recent years.
"""
#print(len(ARTICLE))

#print(summarizer(ARTICLE, max_length=118, min_length=30, do_sample=False))

# +
import string
s = "@Maria: Do you mean the Nook? Be careful, books you buy for the Kindle are for that piece of electronics, and vice versa. I love my Kindle, there are people that swear by the Nook. They like the color screen.Me? I want an ereader that is a reader-- so I dont need color. The kindle battery lasts longer, and the unit isnt as heavy, which can make a difference after reading for a few hours. :) " # Sample string 
out = s.translate(str.maketrans('', '', string.punctuation))


doc = nlp(s.replace("@", ""))
print()
print('Entities:   ', get_entities(doc))
print('Countries   :',  get_countries(doc))
print('PERSON   :',  get_persons(doc))


# +
from transformers import AutoTokenizer, AutoModelForQuestionAnswering
import torch

tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")


# -

def answer_question(question):
    inputs = tokenizer(question, ARTICLE, add_special_tokens=True, return_tensors="pt")
    input_ids = inputs["input_ids"].tolist()[0]
    
    text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
    answer_start_scores, answer_end_scores = model(**inputs)
    
    answer_start = torch.argmax(
        answer_start_scores
    )
    answer_end = torch.argmax(answer_end_scores) + 1
    
    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(input_ids[answer_start:answer_end]))
    return answer


answer_question("How we did to analyze what has been done?")

# +
import numpy as np
from annoy import AnnoyIndex

p = '../dataset/glove.6B.200d.txt'

class PreTrainedEmbeddings(object):
    def __init__(self, word_to_index, word_vectors):
        """
        Args:
            word (str)
        """
        self.word_to_index = word_to_index
        self.word_vectors = word_vectors
        self.index_to_word = { v: k for k, v in self.word_to_index.items() }
        self.index = AnnoyIndex(len(word_vectors[0]), metric='euclidean')
        
        for _, i in self.word_to_index.items():
            self.index.add_item(i, self.word_vectors[i])
        self.index.build(100)
        
    @classmethod
    def from_embeddings_file(cls, embedding_file):
        word_to_index = {}
        word_vectors = []
        with open(embedding_file) as fp:
            for line in fp.readlines():
                line = line.split(" ")
                word = line[0]
                vec = np.array([ float(x) for x in line[1:] ])
                
                word_to_index[word] = len(word_to_index)
                word_vectors.append(vec)
        return cls(word_to_index, word_vectors)
    
    def get_embedding(self, word):
        """
        Args:
            word (str)
        Returns:
            numpy.array
        """
        return self.word_vectors[self.word_to_index[word]]

    def get_closest_to_vector(self, vector, n=1):
        """Given a vector, return its n nearest neighbors
        Args:
            vector (np.array)
            n (int) -> the number of neighbors to return
        Returns:
            [str,str...] -> words nearest to the given vector
        """
        nn_indices = self.index.get_nns_by_vector(vector, n)
        return [self.index_to_word[neighbor] for neighbor in nn_indices]
    
    def compute_analogy(self, word1, word2, word3):
        """
        Args:
            word1 (str)
            word2 (str)
            word3 (str)
        Returns:
            closest_words (list) -> a list of nearest neighbors
        """
        vec1 = self.get_embedding(word1)
        vec2 = self.get_embedding(word2)
        vec3 = self.get_embedding(word3)
        
        # Symple hypothesis: Analogy is a spatial relationship
        spatial_relationship = vec2 - vec1
        vec4 = vec3 + spatial_relationship
        
        closest_words = self.get_closest_to_vector(vec4, n=4)
        existng_words = set([word1, word2, word3])
        closest_words = [word for word in closest_words if word not in existng_words]
        
        return closest_words
    
    def compute_missing_word(self, text):
        """
        Args:
            text (str) -> Text containing blank spaces
        """
        pass


# -

embeddings = PreTrainedEmbeddings.from_embeddings_file(p)

# +
word1, word2, word3 = 'man', 'doctor', 'woman'
closest_words = embeddings.compute_analogy(word1, word2, word3)

print(closest_words)
for word4 in closest_words:
    print("{} : {} :: {} : {}".format(word1, word2, word3, word4))

# +
closest_words = embeddings.compute_and_print_analogy('fast', 'fastest', 'young')

for word4 in closest_words:
    print("{} : {} :: {} : {}".format(word1, word2, word3, word4))

# +
closest_words = embeddings.compute_and_print_analogy('talk', 'communicate', 'read')

for word4 in closest_words:
    print("{} : {} :: {} : {}".format(word1, word2, word3, word4))

# +
from tqdm.notebook import trange, tqdm
from time import sleep

for i in trange(3, desc='1st loop'):
    embeddings = PreTrainedEmbeddings.from_embeddings_file(p)

# +
from nltk.tokenize import TweetTokenizer

tokenizer = TweetTokenizer()
#tokenizer.tokenize("A Jupyter widget could not be displayed because the widget state could not be found. This could happen if the kernel storing the widget is no longer available, or if the widget state was not saved in the notebook. You may be able to create the widget by running the appropriate cells.")

# +
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

class PositionalEncoding(nn.Module):
    def __init__(self, d_model, dropout=0.1, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:x.size(0), :]
        return self.dropout(x)



# -

class TransformerModel(nn.Module):

    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
        super(TransformerModel, self).__init__()
        from torch.nn import TransformerEncoder, TransformerEncoderLayer
        self.model_type = 'Transformer'
        self.src_mask = None
        self.pos_encoder = PositionalEncoding(ninp, dropout)
        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.encoder = nn.Embedding(ntoken, ninp)
        self.ninp = ninp
        self.decoder = nn.Linear(ninp, ntoken)

        self.init_weights()

    def _generate_square_subsequent_mask(self, sz):
        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
        return mask

    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, src):
        if self.src_mask is None or self.src_mask.size(0) != src.size(0):
            device = src.device
            mask = self._generate_square_subsequent_mask(src.size(0)).to(device)
            self.src_mask = mask

        src = self.encoder(src) * math.sqrt(self.ninp)
        src = self.pos_encoder(src)
        output = self.transformer_encoder(src, self.src_mask)
        output = self.decoder(output)
        return output


# +
import torchtext
from torchtext.data.utils import get_tokenizer

TEXT = torchtext.data.Field(tokenize=get_tokenizer("basic_english"),
                           init_token='<sos>',
                           eos_token='<eos>',
                           lower=True)

train_txt, val_txt, test_txt = torchtext.datasets.WikiText2.splits(TEXT)
TEXT.build_vocab(train_txt)
device = torch.device("cuda" if torch.cuda.is_available else "cpu")


# +
def batchify(data, bsz):
    data = TEXT.numericalize([data.examples[0].text])
    nbatch = data.size(0) // bsz
    data = data.narrow(0, 0, nbatch * bsz)
    data = data.view(bsz, -1).t().contiguous()
    return data.to(device)

batch_size = 20
eval_batch_size = 10
train_data = batchify(train_txt, batch_size)
val_data = batchify(val_txt, eval_batch_size)
test_data = batchify(test_txt, eval_batch_size)
# -

bptt = 35
def get_batch(source, i):
    seq_len = min(bptt, len(source) - 1 - i)
    data = source[i:i+seq_len]
    target = source[i+1:i+1+seq_len].view(-1)
    return data, target



# +
ntokens = len(TEXT.vocab.stoi) # the size of vocabulary
emsize = 200 # embedding dimension
nhid = 200 # the dimension of the feedforward network model in nn.TransformerEncoder
nlayers = 2 # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder
nhead = 2 # the number of heads in the multiheadattention models
dropout = 0.2 # the dropout value
model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device)

criterion = nn.CrossEntropyLoss()
lr = 5.0 # learning rate
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)

import time
def train():
    model.train() # Turn on the train mode
    total_loss = 0.
    start_time = time.time()
    ntokens = len(TEXT.vocab.stoi)
    for batch, i in enumerate(range(0, train_data.size(0) - 1, bptt)):
        data, targets = get_batch(train_data, i)
        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output.view(-1, ntokens), targets)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
        optimizer.step()

        total_loss += loss.item()
        log_interval = 200
        if batch % log_interval == 0 and batch > 0:
            cur_loss = total_loss / log_interval
            elapsed = time.time() - start_time
            print('| epoch {:3d} | {:5d}/{:5d} batches | '
                  'lr {:02.2f} | ms/batch {:5.2f} | '
                  'loss {:5.2f} | ppl {:8.2f}'.format(
                    epoch, batch, len(train_data) // bptt, scheduler.get_last_lr()[0],
                    elapsed * 1000 / log_interval,
                    cur_loss, math.exp(cur_loss)))
            total_loss = 0
            start_time = time.time()

def evaluate(eval_model, data_source):
    eval_model.eval() # Turn on the evaluation mode
    total_loss = 0.
    ntokens = len(TEXT.vocab.stoi)
    predictions = []
    with torch.no_grad():
        for i in range(0, data_source.size(0) - 1, bptt):
            data, targets = get_batch(data_source, i)
            output = eval_model(data)
            output_flat = output.view(-1, ntokens)
            total_loss += len(data) * criterion(output_flat, targets).item()
    return total_loss / (len(data_source) - 1)


# +
best_val_loss = float("inf")
epochs = 3
best_model = None

for epoch in range(1, epochs + 1):
    epoch_start_time = time.time()
    train()
    val_loss = evaluate(model, val_data)
    print('-' * 89)
    print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} |'
         'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time),
                                   val_loss, math.exp(val_loss)))
    print('-' * 89)
    
    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_model = model
    
    scheduler.step()
# -

test_loss = evaluate(best_model, test_data)
print("=" * 89)
print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
    test_loss, math.exp(test_loss)))
print("=" * 89)

# +
from string import punctuation

vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}

def tokenize_review(test_review):
    test_review = test_review.lower() # lowercase
    # get rid of punctuation
    test_text = ''.join([c for c in test_review if c not in punctuation])

    # splitting by spaces
    test_words = test_text.split()
    print(test_words)

    # tokens
    test_ints = []
    test_ints.append([vocab_to_int[word] for word in test_words])

    return test_ints

def predict(input_word):
    best_model.eval()
    
    word_ints = tokenize_review(input_word)
    
predict("Hi, there how")
