<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive
!pip install py-automl
#@title Modul Preprocessing { vertical-output: true }
dataset1 = "/content/drive/MyDrive/dataset/dataset_2.xlsx" #@param {type:"string"}
dataset2 = "/content/drive/MyDrive/dataset/dataset_2_split.xlsx" #@param {type:"string"}
indeks1 = "2005 - Grid 3D" #@param {type:"string"}
indeks2 = "Requirement Statement" #@param {type:"string"}
indeks3 = "ID" #@param {type:"string"}

import pandas as pd
import nltk
from tabulate import tabulate
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize 
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.collocations import BigramCollocationFinder, BigramAssocMeasures

# download nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')

stemming = PorterStemmer()
stops = set(stopwords.words("english"))
lem = WordNetLemmatizer()


class cleaningProcess:
  
  def __init__(self, data):
      self.index0 = data

  # fungsi ini untuk mengecek data preprocessing
  def preprocessing(self):
      x1 = pd.ExcelFile(self.index0)
      dfs = {sh:x1.parse(sh) for sh in x1.sheet_names}
      return dfs

  # fungsi ini digunakan untuk mengecek data secara keseluruhan dataset tertentu
  def fulldataset(self, index1):
      x1 = pd.ExcelFile(self.index0)
      dfs = {sh:x1.parse(sh) for sh in x1.sheet_names}[index1]
      return dfs

  def clean_text(self, raw_text):
      text = raw_text.lower()
      tokens = nltk.word_tokenize(text)
      token_words = [w for w in tokens if w.isalpha()]
      lemma_words = [lem.lemmatize(w) for w in token_words]
      meaningful_words = [w for w in lemma_words if not w in stops]
      joined_words = ( " ".join(meaningful_words))
      return joined_words

  # cleaning text
  def apply_cleaning_function_to_list(self, X):
      cleaned_X = [cleaningProcess.clean_text(self, element) for element in X]
      return cleaned_X

if __name__ == "__main__":

    myCleaning1 = cleaningProcess(dataset1)
    list_req1 = myCleaning1.fulldataset(indeks1)[indeks2]
    id_req1 = myCleaning1.fulldataset(indeks1)[indeks3]
    cleaned_text1 = myCleaning1.apply_cleaning_function_to_list(list_req1)
    data_raw1 = pd.DataFrame(cleaned_text1, index= id_req1, columns= ['cleaned'])
    print("cleaning data {}".format(dataset1))
    print(tabulate(data_raw1, headers = 'keys', tablefmt = 'psql'))   
    print("\n")
    myCleaning2 = cleaningProcess(dataset2)
    list_req2 = myCleaning2.fulldataset(indeks1)[indeks2]
    id_req2 = myCleaning2.fulldataset(indeks1)[indeks3]
    cleaned_text2 = myCleaning2.apply_cleaning_function_to_list(list_req2)
    data_raw2 = pd.DataFrame(cleaned_text2, index= id_req2, columns= ['cleaned'])
    print("cleaning data {}".format(dataset2))
    print(tabulate(data_raw2, headers = 'keys', tablefmt = 'psql'))   
    

#@title Modul Similarity
#@markdown Berikut ini adalah modul similarity yang digunakan meliputi
#@markdown 1. Cosine similarity
#@markdown 2. Jaccard similarity
#@markdown 3. Levenshtein similarity
#@markdown 4. TFIDF
#@markdown 5. VSM
#@markdown 6. PMI
parameter = "cosine" #@param ["cosine", "jaccard", "levenshtein", "tfidf", "vsm", "pmi"]

import nltk, math, statistics
import re, collections
import numpy as np
import string #allows for format()
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

class similarityMeasurement:
  
  def __init__(self):
      pass
      
  # lavenstein
  def similarity_levenshtein(self, s1, s2):
      if len(s1) < len(s2):
          return similarityMeasurement.similarity_levenshtein(self, s2, s1)
      if len(s2) == 0:
          return len(s1)

      previous_row = range(len(s2) + 1)
      for i, c1 in enumerate(s1):
          current_row = [i + 1]
          for j, c2 in enumerate(s2):
              insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
              deletions = current_row[j] + 1       # than s2
              substitutions = previous_row[j] + (c1 != c2)
              current_row.append(min(insertions, deletions, substitutions))
          previous_row = current_row    
      return previous_row[-1]

  def measurement_levenshtein(self, data1, data2):
      data = [[similarityMeasurement.similarity_levenshtein(self, num, angka) for angka in data2] for num in data1]
      return data

  # Cosine Similarity 
  def similarity_cosine(self, data1, data2):
      X_list = word_tokenize(data1) 
      Y_list = word_tokenize(data2) 
      sw = stopwords.words('english') 
      l1 =[];l2 =[] 
      X_set = {w for w in X_list if not w in sw} 
      Y_set = {w for w in Y_list if not w in sw} 
      rvector = X_set.union(Y_set) 
      for w in rvector: 
        if w in X_set: l1.append(1) # create a vector 
        else: l1.append(0) 
        if w in Y_set: l2.append(1) 
        else: l2.append(0) 
      c = 0

      # cosine formula 
      for i in range(len(rvector)): 
          c+= l1[i]*l2[i] 
      cosine = c / float((sum(l1)*sum(l2))**0.5) 
      return cosine 

  def measurement_cosine(self, data1, data2):
      data = [[similarityMeasurement.similarity_cosine(self, num, angka) for angka in data2] for num in data1]
      return data

  # jaccard similarity
  def similarity_jaccard(self, list1, list2):
      intersection = len(list(set(list1).intersection(list2)))
      union = (len(list1) + len(list2)) - intersection
      return float(intersection) / union

  def measurement_jaccard(self, data1, data2):
      data = [[similarityMeasurement.similarity_jaccard(self, num, angka) for angka in data2] for num in data1]
      return data

  # term frequency - inverse document frequency
  def tfidf(self, data):
      vect = TfidfVectorizer(stop_words='english')
      tfidf_matrix = vect.fit_transform(data)
      return tfidf_matrix.toarray()

  # vsm similarity
  def vsm_similarity(self, data1, data2):
      hasil1 = similarityMeasurement.tfidf(self, data1)
      hasil2 = similarityMeasurement.tfidf(self, data2)
      vsm = cosine_similarity(hasil1, hasil2)
      return vsm

  def pmi_measurement(self, text1, text2):
      stopwords_ = set(stopwords.words('english'))
      words1 = [word.lower() for word in text1.split() if len(word) > 2 and word not in stopwords_]
      words2 = [word.lower() for word in text2.split() if len(word) > 2 and word not in stopwords_]

      finder = BigramCollocationFinder.from_words(words1+words2)
      bgm = BigramAssocMeasures()
      score = bgm.mi_like
      collocations = {'_'.join(bigram): pmi for bigram, pmi in finder.score_ngrams(score)}
      return collocations

  def pmi_jumlah(self, text1, text2):
      stopwords_ = set(stopwords.words('english'))
      words1 = [word.lower() for word in text1.split() if len(word) > 2 and word not in stopwords_]
      words2 = [word.lower() for word in text2.split() if len(word) > 2 and word not in stopwords_]
      finder = BigramCollocationFinder.from_words(words1+words2)
      bgm = BigramAssocMeasures()
      score = bgm.mi_like
      total_pmi = sum([math.log(pmi) for bigram, pmi in finder.score_ngrams(score)])
      return total_pmi

  def measurement_pmi_jumlah(self, data1, data2):
      data = [[similarityMeasurement.pmi_jumlah(self, num, angka) for angka in data2] for num in data1]
      return data

  def co_occurrence(self, sentences, window_size= 2):
        d = collections.defaultdict(int)
        vocab = set()
        for text in sentences:
            # preprocessing (use tokenizer instead)
            text = text.lower().split()
            # iterate over sentences
            for i in range(len(text)):
                token = text[i]
                vocab.add(token)  # add to vocab
                next_token = text[i+1 : i+1+window_size]
                for t in next_token:
                    key = tuple( sorted([t, token]) )
                    d[key] += 1

        # formulate the dictionary into dataframe
        vocab = sorted(vocab) # sort vocab
        df = pd.DataFrame(data=np.zeros((len(vocab), len(vocab)), dtype=np.int16), index=vocab, columns=vocab)
        for key, value in d.items():
            df.at[key[0], key[1]] = value
            df.at[key[1], key[0]] = value
        return df

if __name__ == "__main__":

  if parameter == "cosine":
      print("Data Parameter {}".format(parameter))
      mySim = similarityMeasurement()
      data_cosine = mySim.measurement_cosine(cleaned_text1, cleaned_text2)
      df_cosine = pd.DataFrame(data_cosine, index= id_req1, columns= id_req2)
      print(tabulate(df_cosine, headers = 'keys', tablefmt = 'psql'))   

  elif parameter == "jaccard":
      print("Data Parameter {}".format(parameter))
      data_jaccard = mySim.measurement_jaccard(cleaned_text1, cleaned_text2)
      df_jaccard = pd.DataFrame(data_jaccard, index= id_req1, columns= id_req2)
      print(tabulate(df_jaccard, headers = 'keys', tablefmt = 'psql'))   

  elif parameter == "levenshtein":
      print("Data Parameter {}".format(parameter))
      data_levenshtein = mySim.measurement_levenshtein(cleaned_text1, cleaned_text2)
      df_levensthein = pd.DataFrame(data_levenshtein, index= id_req1, columns= id_req2)
      print(tabulate(df_levensthein, headers = 'keys', tablefmt = 'psql'))   

  elif parameter == "tfidf":
      print("Data Parameter {}".format(parameter))
      data_tfidf1 = mySim.tfidf(cleaned_text1)
      df_tfidf1 = pd.DataFrame(data_tfidf1, index= id_req1)
      print(tabulate(df_tfidf1, headers = 'keys', tablefmt = 'psql'))   

      data_tfidf2 = mySim.tfidf(cleaned_text2)
      df_tfidf2 = pd.DataFrame(data_tfidf2, index= id_req2)
      print(tabulate(df_tfidf2, headers = 'keys', tablefmt = 'psql'))   

  elif parameter == "vsm":
      print("Data Parameter {}".format(parameter))
      data_vsm = mySim.vsm_similarity(cleaned_text1, cleaned_text2)
      df_vsm = pd.DataFrame(data_vsm, index= id_req1, columns= id_req2)
      print(tabulate(df_vsm, headers = 'keys', tablefmt = 'psql'))   

  if parameter == "pmi":
      print("Data Parameter {}".format(parameter))
      data_pmi_jumlah = mySim.measurement_pmi_jumlah(cleaned_text1, cleaned_text2)
      df_pmi_jumlah = pd.DataFrame(data_pmi_jumlah, index= id_req1, columns= id_req2)
      print(tabulate(df_pmi_jumlah, headers = 'keys', tablefmt = 'psql'))   



#@title Modul Visualisasi dan Threshold
parameter = "vsm" #@param ["cosine", "jaccard", "levenshtein", "tfidf", "vsm", "pmi"]
#@markdown Berikut ini adalah deskripsi singkat, bagaimana threshold mempengaruhi nilai statistik dari setiap data yang akan dilihat
%matplotlib notebook
from sklearn.decomposition import PCA
import altair as alt
import matplotlib.pyplot as plt
import seaborn as sns


class visualThres:
  
  def __init__(self):
      pass

  # visualisasi dan threshold
  def threshold_value(self, threshold, data):
      dt = data.values >= threshold
      dt1 = pd.DataFrame(dt, index= data.index, columns= data.columns)
      mask = dt1.isin([True])
      dt3 = dt1.where(mask, other= 0)
      mask2 = dt3.isin([False])
      th_cosine1 = dt3.where(mask2, other= 1)
      return th_cosine1

  def sent_PCA(self, sentence, n = 2):
      pca = PCA(n_components = n)
      pca.fit(np.array(sentence).transpose())
      variance = np.array(pca.explained_variance_ratio_)
      words = []
      for _ in range(n):
          idx = np.argmax(variance)
          words.append(np.amax(variance) * sentence[idx])
          variance[idx] = 0
      return np.sum(words, axis = 0)

  def visualisasi(self, data):
      import matplotlib.pyplot as plt
      from sklearn.decomposition import PCA
      pca = PCA(n_components=len(data))
      my_pca = pca.fit_transform(data)
      plt.scatter(my_pca[:,0], my_pca[:,1])
      for i, word in enumerate(data.index):
          plt.annotate(word, xy=(my_pca[i,0], my_pca[i,1]))
          plt.xlabel('widht')
          plt.ylabel('height')
      plt.title('Visualisasi')
      plt.show()

  def npstat(self, data):
      maximum = np.max(data)
      minimum = np.min(data)
      mean = np.mean(data)
      variance = np.var(data)
      median = np.median(data)
      standar_deviasi = np.std(data)
      dataBaru = [maximum, minimum, mean, variance, median, standar_deviasi]
      return dataBaru

if __name__ == "__main__":
     id_default = ['maks', 'min', 'mean', 'var', 'med', 'std_dev']
     id_kolom = ['stat_value']
     if parameter == "cosine":
        print("\nstatistic {}".format(parameter))
        myVisual = visualThres()
        stat_cosine = myVisual.npstat(data_cosine)
        th_cosine = myVisual.threshold_value(stat_cosine[2], df_cosine)
        stat_th_cosine = myVisual.npstat(th_cosine.values)
        stat_cosine_df = pd.DataFrame(stat_th_cosine, index= id_default, columns= id_kolom)
        print(tabulate(stat_cosine_df, headers = 'keys', tablefmt = 'psql'))   

     elif parameter == "jaccard":
        print("\nstatistic {}".format(parameter))
        stat_jaccard = myVisual.npstat(data_jaccard)
        th_jaccard = myVisual.threshold_value(stat_jaccard[2], df_jaccard)
        stat_th_jaccard = myVisual.npstat(th_jaccard.values)
        stat_jaccard_df = pd.DataFrame(stat_th_jaccard, index= id_default, columns= id_kolom)
        print(tabulate(stat_jaccard_df, headers = 'keys', tablefmt = 'psql'))   

     elif parameter == "levenshtein":
        print("\nstatistic {}".format(parameter))
        stat_levenshtein = myVisual.npstat(data_levenshtein)
        th_levenshtein = myVisual.threshold_value(stat_levenshtein[2], df_levensthein)
        stat_th_levenshtein = myVisual.npstat(th_levenshtein.values)
        stat_levenshtein_df = pd.DataFrame(stat_th_levenshtein, index= id_default, columns= id_kolom)
        print(tabulate(stat_levenshtein_df, headers = 'keys', tablefmt = 'psql'))   

     elif parameter == "tfidf":
        print("\nstatistic {}".format(parameter))
        stat_tfidf1 = myVisual.npstat(data_tfidf1)
        th_tfidf1 = myVisual.threshold_value(stat_tfidf1[2], df_tfidf1)
        stat_th_tfidf1 = myVisual.npstat(th_tfidf1.values)
        stat_tfidf1_df = pd.DataFrame(stat_th_tfidf1, index= id_default, columns= id_kolom)
        print(tabulate(stat_tfidf1_df, headers = 'keys', tablefmt = 'psql'))  

     elif parameter == "vsm":
        print("\nstatistic {}".format(parameter))
        stat_vsm = myVisual.npstat(data_vsm)
        th_vsm = myVisual.threshold_value(stat_vsm[2], df_vsm)
        stat_th_vsm = myVisual.npstat(th_vsm.values)
        stat_vsm_df = pd.DataFrame(stat_vsm, index= id_default, columns= id_kolom)
        print(tabulate(stat_vsm_df, headers = 'keys', tablefmt = 'psql'))  

    #  elif parameter == "pmi": #masih error
    #     # df_pmi_jumlah = pd.DataFrame(data_pmi_jumlah)
    #     th_pmi_jumlah = myVisual.threshold_value(stat_pmi_jumlah[0], df_pmi_jumlah)
    #     stat_th_pmi_jumlah = npstat(th_pmi_jumlah.values)
    #     print("\nstatistic pmi")
    #     stat_pmi_df = pd.DataFrame(stat_th_pmi_jumlah, index= id_default, columns= id_kolom)
    #     print(tabulate(stat_pmi_df, headers = 'keys', tablefmt = 'psql'))  
#@title Modul Evaluasi
#@markdown Berikut ini adalah modul evaluasi, menunjukkan data evaluasi antara groundtruth dan data testing
parameter = "cosine" #@param ["cosine", "jaccard", "levenshtein", "tfidf", "vsm", "pmi"]

import pandas as pd
import numpy as np
from pyAutoML.ml import ML,ml, EncodeCategorical
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split

class pengukuranEvaluasi:
  def __init__(self, dataPertama, dataKedua):
      self.data1 = dataPertama
      self.data2 = dataKedua

  def ukur_evaluasi(self):
      X_train, X_test, y_train, y_test = train_test_split(self.data1, self.data2, test_size=0.3,random_state=109) # 70% training and 30% test
      y_train = y_train.argmax(axis= 1)
      X = X_train
      Y = y_train
      Y = EncodeCategorical(Y)
      size = 0.4
      return ML(X, Y, size, SVC(), RandomForestClassifier(), DecisionTreeClassifier(), KNeighborsClassifier(), LogisticRegression(max_iter = 7000))      

if __name__ == "__main__":
  if parameter == 'levenshtein':
    myEvaluasi = pengukuranEvaluasi(data_levenshtein, th_levenshtein.values) # normal
    myEvaluasi.ukur_evaluasi()
  elif parameter == 'cosine':
    myEvaluasi = pengukuranEvaluasi(data_cosine, th_cosine.values) #evaluasi
    myEvaluasi.ukur_evaluasi()    
  elif parameter == 'jaccard':
    myEvaluasi = pengukuranEvaluasi(data_jaccard, th_jaccard.values) #evaluasi
    myEvaluasi.ukur_evaluasi()
  elif parameter == 'tfidf':
    myEvaluasi = pengukuranEvaluasi(data_tfidf1, th_tfidf1.values) #evaluasi
    myEvaluasi.ukur_evaluasi()
  elif parameter == 'vsm':
    myEvaluasi = pengukuranEvaluasi(data_vsm, th_vsm.values) # evaluasi 
    myEvaluasi.ukur_evaluasi()
  # elif parameter == 'pmi':
  #   myEvaluasi = pengukuranEvaluasi(data_pmi_jumlah, th_pmi_jumlah.values) # evaluasi
  #   myEvaluasi.ukur_evaluasi()<jupyter_output>____________________________________________________
.....................Py-AutoML......................
____________________________________________________
SVC ______________________________ 

Accuracy Score for SVC is 
0.8


Confusion Matrix for SVC is 
[[4 0]
 [1 0]]


Classification Report for SVC is 
              precision    recall  f1-score   support

           0       0.80      1.00      0.89         4
           1       0.00      0.00      0.00         1

    accuracy                           0.80         5
   macro avg       0.40      0.50      0.44         5
weighted avg       0.64      0.80      0.71         5



____________________________________________________
RandomForestClassifier ______________________________ 

Accuracy Score for RandomForestClassifier is 
0.8


Confusion Matrix for RandomForestClassifier is 
[[4 0]
 [1 0]]


Classification Report for RandomForestClassifier is 
              precision    recall  f1-score   support

           0       0.80     [...]
StarcoderdataJupyterStructuredCleanDedup