# -*- coding: utf-8 -*-
"""Opinion_mining.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/10FDuEHaj0sM7bTxQqaK7jENJa5dFTXeY

# Análisis de sentimiento
"""

!pip install beautifulsoup4
!pip install google-search-results-serpwow
!pip install google-search-results-serpwow --upgrade

# Commented out IPython magic to ensure Python compatibility.
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
# %matplotlib inline
from scipy import stats
import pandas as pd 
import multiprocessing
import random

from matplotlib import rc
from sklearn.preprocessing import scale
from sklearn.preprocessing import OneHotEncoder
import timeit
import itertools
import seaborn as sns
from collections import Counter
import operator

import requests
import json
import csv

from google.colab import drive
drive.mount('/content/drive')

review_results = []

urls = [
          #Tiradores: Tirador para muebles, cajón, armario
        'https://www.amazon.es/Emuca-9160951-Furniture-Handle-Nickel/product-reviews/B075YV44YJ/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.it/Emuca-9160951-Furniture-Handle-Nickel/product-reviews/B075YV44YJ/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.fr/Emuca-9160951-Furniture-Handle-Nickel/product-reviews/B075YV44YJ/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.co.uk/Emuca-9160951-Furniture-Handle-Nickel/product-reviews/B075YV44YJ/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',

          #Cajones: Kit de cajón para cocina y baño
        'https://www.amazon.es/Emuca-3018712-Caj%C3%B3n-Cierre-Blanco/product-reviews/B079H95LXG/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.it/Emuca-3018712-Caj%C3%B3n-Cierre-Blanco/product-reviews/B079H95LXG/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.fr/Emuca-3018712-Caj%C3%B3n-Cierre-Blanco/product-reviews/B079H95LXG/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.co.uk/Emuca-3018712-Caj%C3%B3n-Cierre-Blanco/product-reviews/B079H95LXG/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        
          #Armarios: Barra de armario abatible
        'https://www.amazon.es/Emuca-7086311-basculante-regulable-830-1150mm/dp/B00XKV5CWW/ref=cm_cr_arp_d_product_top?ie=UTF8',
        'https://www.amazon.it/Emuca-7086311-basculante-regulable-830-1150mm/dp/B00XKV5CWW/ref=cm_cr_arp_d_product_top?ie=UTF8',
        'https://www.amazon.fr/Emuca-7086311-basculante-regulable-830-1150mm/dp/B00XKV5CWW/ref=cm_cr_arp_d_product_top?ie=UTF8',
        'https://www.amazon.co.uk/Emuca-7086311-basculante-regulable-830-1150mm/dp/B00XKV5CWW/ref=cm_cr_arp_d_product_top?ie=UTF8',

          #Bases: Ruedas
        'https://www.amazon.es/2037121-ruedas-pivotantes-mueble-rodamientos/product-reviews/B01DDE7GHE/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.it/2037121-ruedas-pivotantes-mueble-rodamientos/product-reviews/B01DDE7GHE/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.fr/2037121-ruedas-pivotantes-mueble-rodamientos/product-reviews/B01DDE7GHE/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.co.uk/2037121-ruedas-pivotantes-mueble-rodamientos/product-reviews/B01DDE7GHE/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',

          #Equipamiento: Escurridor de cocina
        'https://www.amazon.es/product-reviews/B01N4SFEM5/ref=acr_dpx_hist_1?ie=UTF8&filterByStar=one_star&reviewerType=all_reviews',
        'https://www.amazon.it/product-reviews/B01N4SFEM5/ref=acr_dpx_hist_1?ie=UTF8&filterByStar=one_star&reviewerType=all_reviews',
        'https://www.amazon.fr/product-reviews/B01N4SFEM5/ref=acr_dpx_hist_1?ie=UTF8&filterByStar=one_star&reviewerType=all_reviews',
        'https://www.amazon.co.uk/product-reviews/B01N4SFEM5/ref=acr_dpx_hist_1?ie=UTF8&filterByStar=one_star&reviewerType=all_reviews',

          #Iluminación: Multienchufe
        'https://www.amazon.es/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.it/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.fr/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.co.uk/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews'

          #Montaje: Paragolpe
        'https://www.amazon.es/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.it/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.fr/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.co.uk/Emuca-5070125-multienchufe-retr%C3%A1ctil-3500W-16A/product-reviews/B01GV0PE26/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews'

          #Montaje: Sistema push para puerta de mueble
        'https://www.amazon.es/Emuca-1275221-Sistema-puerta-amortiguador/product-reviews/B0771STHRB/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.es/Emuca-1275221-Sistema-puerta-amortiguador/product-reviews/B0771STHRB/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.es/Emuca-1275221-Sistema-puerta-amortiguador/product-reviews/B0771STHRB/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews',
        'https://www.amazon.es/Emuca-1275221-Sistema-puerta-amortiguador/product-reviews/B0771STHRB/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews'


]

for url in urls:
  params = {
    'api_key': '9A7BE26DBE9D44A3924748F7F8CC1C35',
    'type': 'reviews',
    'url': url,
    'output': 'json'
    }

  api_result = requests.get('https://api.rainforestapi.com/request', params).json()

  if 'reviews' in api_result:
    review_results += api_result['reviews']

review_results

with open('opinion_mining.csv', 'w', encoding='utf-8') as file:
    fields = ['title', 'body', 'rating', 'review_country', 'verified_purchase', 'id']
    writer = csv.DictWriter(file, fieldnames=fields)
    writer.writeheader()
    for review in review_results:
        writer.writerow({
            'title' : review['title'] if 'title' in review else None,
            'body': review['body'] if 'body' in review else None,
            'rating': review['rating'] if 'rating' in review else None, 
            'review_country': review['review_country'] if 'review_country' in review else None, 
            'verified_purchase': review['verified_purchase'] if 'verified_purchase' in review else None,
            'id': review['id'] if 'id' in review else None
            })

df = pd.read_csv("/content/opinion_mining.csv")
df

df.groupby(by='review_country').size()

df.groupby(by='rating').size()

df.to_csv('/content/drive/My Drive/TFM/03_DATASETS/opinion_mining_all_reviews.csv',sep=';',decimal=',')

"""# Split by country and language"""

df_es=df[df['review_country']=='es']
df_it=df[df['review_country']=='it']
df_fr=df[df['review_country']=='fr']
df_gb=df[df['review_country']=='gb']
df_gb.sort_values(by='rating',ascending=True).head(15)

"""### Import packages"""

#Import packages
import re, string
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer 
from textblob import TextBlob

nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')

"""### Defining functions

We remove all capital letters, punctuations, emojis, links, etc. Basically, removing all that is not words or numbers.
"""

#The function
def clean_text(text):     
    text = text.lower()
    text = re.sub('@', '', text)
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub(r"[^a-zA-Z ]+", "", text)
    
    #Tokenize the data (into words, which means breaking up every comment into a group of individual words.)
    text = nltk.word_tokenize(text)
    #Remove stopwords (which are words that don’t add value to a comment, like “the”, “a”, “and”, etc.)
    text = [w for w in text if w not in sw]
    return text

"""We lemmatize the text, which is the process of grouping together the inflected forms of a word so they can be analyzed as a single item, since they have a similar meaning (walking becomes walk, officers becomes officer, etc.)."""

#Lemmatizer
lemmatizer = WordNetLemmatizer()
def lem(text):
    text = [lemmatizer.lemmatize(t) for t in text]
    text = [lemmatizer.lemmatize(t, 'v') for t in text]
    return text

"""### Reviews UK

We define the text language we will analyze
"""

sw = stopwords.words('english')

"""We apply the function to the data"""

df_gb['body'] = df_gb['body'].apply(lambda x: clean_text(x))

"""We lemmatize the text, which is the process of grouping together the inflected forms of a word so they can be analyzed as a single item, since they have a similar meaning (walking becomes walk, officers becomes officer, etc.)."""

df_gb['body'] = df_gb['body'].apply(lambda x: lem(x))

"""We remove all empty comments from the data (some people just comment emojis, punctuations or things like that)."""

df_gb.reset_index(inplace=True, drop=True)

len(df_gb)

#Remove all empty comments
empty_comment = ""
for i in range(len(df_gb)):
    if df_gb['body'] [i]==empty_comment:
        df_gb=df_gb.drop(i)
df_gb=df_gb.reset_index(drop=True)

"""**Word Frequency** Let’s begin by looking at the word frequency, i.e. what words are repeated most often in the comments, using the FreqDist function from nltk."""

#From lists of comments to a single list containing all words      
all_words=[]        
for i in range(len(df_gb)):
  all_words = all_words + df_gb['body'][i]
#Get word frequency        
nlp_words = nltk.FreqDist(all_words)
plot1 = nlp_words.plot(20, color='salmon', title='Word Frequency')

"""Let’s look for the most frequent bigrams, which means the most frequent pair of words that are next to each other in a comment."""

#Bigrams
bigrm = list(nltk.bigrams(all_words))
words_2 = nltk.FreqDist(bigrm)
words_2.plot(20, color='salmon', title='Bigram Frequency')

"""**Polarity**
Using Textblob’s sentiment function, we can look at the polarity of a comment, which is a float number ranging between -1 and 1 that is meant to represent if a comment is positive (1) or negative (-1). For instance, the sentence “Textblob is great” has a polarity of 0.4.
Here is how to get polarity for every comment and the distribution of said polarity:
"""

#Get sentiment from comments
df_gb['body'] = [str(thing) for thing in df_gb['body']]
sentiment_gb = []
for i in range(len(df_gb)):
    blob = TextBlob(df_gb['body'][i])
    for sentence in blob.sentences:
        sentiment_gb.append(sentence.sentiment.polarity)
df_gb['sentiment']=sentiment_gb
#Plot
df_gb['sentiment'].plot.hist(color='salmon', title='Comments Polarity English')

"""### Reviews Spain"""

sw = stopwords.words('spanish')

#The function
def clean_text(text):     
    text = text.lower()
    text = re.sub('@', '', text)
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub(r"[^a-zA-Z ]+", "", text)
    
    #Tokenize the data (into words, which means breaking up every comment into a group of individual words.)
    text = nltk.word_tokenize(text)
    #Remove stopwords (which are words that don’t add value to a comment, like “the”, “a”, “and”, etc.)
    text = [w for w in text if w not in sw]
    return text

df_es['body'] = df_es['body'].apply(lambda x: clean_text(x))

df_es['body'] = df_es['body'].apply(lambda x: lem(x))

df_es.reset_index(inplace=True, drop=True)

#Remove all empty comments
empty_comment = ""
for i in range(len(df_es)):
  if df_es['body'][i]==empty_comment:
    df_es=df_es.drop(i)
df_es=df_es.reset_index(drop=True)

#From lists of comments to a single list containing all words      
all_words=[]        
for i in range(len(df_es)):
  all_words = all_words + df_es['body'][i]
#Get word frequency        
nlp_words = nltk.FreqDist(all_words)
plot1 = nlp_words.plot(20, color='salmon', title='Word Frequency')

#Bigrams
bigrm = list(nltk.bigrams(all_words))
words_2 = nltk.FreqDist(bigrm)
words_2.plot(20, color='salmon', title='Bigram Frequency')

#Get sentiment from comments
df_es['body'] = [str(thing) for thing in df_es['body']]
sentiment_es = []
for i in range(len(df_es)):
    blob = TextBlob(df_es['body'][i])
    for sentence in blob.sentences:
        sentiment_es.append(sentence.sentiment.polarity)
df_es['sentiment']=sentiment_es
#Plot
df_es['sentiment'].plot.hist(color='salmon', title='Comments Polarity Spanish')

"""### Reviews France"""

sw = stopwords.words('french')

#The function
def clean_text(text):     
    text = text.lower()
    text = re.sub('@', '', text)
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub(r"[^a-zA-Z ]+", "", text)
    
    #Tokenize the data (into words, which means breaking up every comment into a group of individual words.)
    text = nltk.word_tokenize(text)
    #Remove stopwords (which are words that don’t add value to a comment, like “the”, “a”, “and”, etc.)
    text = [w for w in text if w not in sw]
    return text

df_fr['body'] = df_fr['body'].apply(lambda x: clean_text(x))

df_fr['body'] = df_fr['body'].apply(lambda x: lem(x))

df_fr.reset_index(inplace=True, drop=True)

#Remove all empty comments
empty_comment = ""
for i in range(len(df_fr)):
  if df_fr['body'][i]==empty_comment:
    df_fr=df_fr.drop(i)
df_fr=df_fr.reset_index(drop=True)

#From lists of comments to a single list containing all words      
all_words=[]        
for i in range(len(df_fr)):
  all_words = all_words + df_fr['body'][i]
#Get word frequency        
nlp_words = nltk.FreqDist(all_words)
plot1 = nlp_words.plot(20, color='salmon', title='Word Frequency')

#Bigrams
bigrm = list(nltk.bigrams(all_words))
words_2 = nltk.FreqDist(bigrm)
words_2.plot(20, color='salmon', title='Bigram Frequency')

#Get sentiment from comments
df_fr['body'] = [str(thing) for thing in df_fr['body']]
sentiment_fr = []
for i in range(len(df_fr)):
    blob = TextBlob(df_fr['body'][i])
    for sentence in blob.sentences:
        sentiment_fr.append(sentence.sentiment.polarity)
df_fr['sentiment']=sentiment_fr
#Plot
df_fr['sentiment'].plot.hist(color='salmon', title='Comments Polarity French')

"""### Reviews Italy"""

sw = stopwords.words('italian')

#The function
def clean_text(text):     
    text = text.lower()
    text = re.sub('@', '', text)
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub(r"[^a-zA-Z ]+", "", text)
    
    #Tokenize the data (into words, which means breaking up every comment into a group of individual words.)
    text = nltk.word_tokenize(text)
    #Remove stopwords (which are words that don’t add value to a comment, like “the”, “a”, “and”, etc.)
    text = [w for w in text if w not in sw]
    return text

df_it['body'] = df_it['body'].apply(lambda x: clean_text(x))

df_it['body'] = df_it['body'].apply(lambda x: lem(x))

df_it.reset_index(inplace=True, drop=True)

#Remove all empty comments
empty_comment = ""
for i in range(len(df_it)):
  if df_it['body'][i]==empty_comment:
    df_it=df_it.drop(i)
df_it=df_it.reset_index(drop=True)

#From lists of comments to a single list containing all words      
all_words=[]        
for i in range(len(df_it)):
  all_words = all_words + df_it['body'][i]
#Get word frequency        
nlp_words = nltk.FreqDist(all_words)
plot1 = nlp_words.plot(20, color='salmon', title='Word Frequency')

#Bigrams
bigrm = list(nltk.bigrams(all_words))
words_2 = nltk.FreqDist(bigrm)
words_2.plot(20, color='salmon', title='Bigram Frequency')

#Get sentiment from comments
df_it['body'] = [str(thing) for thing in df_it['body']]
sentiment_it = []
for i in range(len(df_it)):
    blob = TextBlob(df_it['body'][i])
    for sentence in blob.sentences:
        sentiment_it.append(sentence.sentiment.polarity)
df_it['sentiment']=sentiment_it
#Plot
df_it['sentiment'].plot.hist(color='salmon', title='Comments Polarity Italian')

"""### Summary"""

df_gb_mean = df_gb['sentiment'].mean()
df_fr_mean = df_fr['sentiment'].mean()
df_it_mean = df_it['sentiment'].mean()
df_es_mean = df_es['sentiment'].mean()

opinion_mining = {'country': ['Great Britain','France','Italy','Spain'],
        'comments_polarity': [df_gb_mean,df_fr_mean,df_it_mean,df_es_mean]
        }

df_opinion_mining = pd.DataFrame(opinion_mining, columns = ['country', 'comments_polarity'])
df_opinion_mining

from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
word_string=" ".join(df['title'].str.lower())
wordcloud = WordCloud(stopwords=STOPWORDS,
                          background_color='white', 
                      max_words=300
                         ).generate(word_string)

plt.clf()
plt.imshow(wordcloud)
plt.axis('off')
plt.show()
plt.figure(figsize=(20, 6),dpi=500)