<jupyter_start><jupyter_text>- Importing the libraries here...<jupyter_code># !python -m spacy download en

# !python -m spacy download en_core_web_lg
# !pip install truecase
import nltk
# nltk.download()
import pandas as pd
import spacy
# from spacy import displacy
# from collections import Counter
import en_core_web_lg
nlp = en_core_web_lg.load()
import re
import truecase
import numpy as np
import ast
import json
# from operator import add 
# from operator import truediv
# !python -m spacy download en_core_web_lg
# python -m spacy download en_core_web_lg
  # tech = pd.read_csv("/content/Technology Summary Matching Project - technologies.csv")
  # summary = pd.read_csv("/content/c22b731c-cff3-4c60-b880-9f42f2fe45cb.csv")
  # output = pd.read_csv("/content/output - output.csv")

# tech = pd.read_csv("Dataset/Technology Summary Matching Project - technologies.csv")
tech = pd.read_csv("Eng_pydictionary_2.csv")

# summary = pd.read_csv("a4f32761-65b8-45e8-99ed-fb8ad4a63f8d.csv")
# summary = pd.read_csv("train_data_original_untagged.csv")
summary = pd.read_csv("Train_Data/Accuracy_Data/Dataset_to_run_on_all/tagged_simanchala_100.csv")

# summary = pd.read_csv("Train_Data/String_Match_non_duplicates - String_Match_non_duplicates.csv")
output = pd.read_csv("Dataset/output - output.csv")
tech.head()
len(tech)

tech.columns = ["index", "title"]
tech.drop(columns = ["index"], axis = 1, inplace=True)
tech.reset_index(inplace = True, drop = True)
for i in range(0, len(summary)):
    if summary["summaries"][i] is not np.nan:
        demo = summary["summaries"][i]
        res = demo.strip('][')
        resl = []
        resl.append(res)
        summary["summaries"][i] = resl
        
    else:
        summary["summaries"][i] = '[]'
        demo = summary["summaries"][i]
        res = demo.strip('][')
        resl = []
        resl.append(res)
        summary["summaries"][i] = resl
# # Count no of lowercase and uppercase in the whole data

# # Python3 program to count upper and 
# # lower case characters without using 
# # inbuilt functions 
# def count_upperlower(df):
#     upper = 0
#     lower = 0
#     lower_list = []
#     upper_list = []
    
#     for ind in df.index:
#         print(ind)
        
#         if df["summaries"][ind][0].islower():
#             lower_list.append(ind)
#             lower+=1
            
#         elif df["summaries"][ind][0].isupper():
#             upper_list.append(ind)
            
#             upper+=1
#     return lower, upper, lower_list, upper_list
# lower_count, upper_count, lower_list, upper_list = count_upperlower(summary)<jupyter_output><empty_output><jupyter_text>Truecasing<jupyter_code>for ind in summary.index:
    summary["summaries"][ind][0] = truecase.get_true_case(summary["summaries"][ind][0])
# # h = nlp('Bachelor of science( B. S) degree in" telecommunications networks and software" program at Politehnica University- electronics, telecommunications and information technology faculty master of science( M. S.) degree in" management of networks and services" program at Politehnica University- electronics, telecommunications and information technology faculty routing and switching skills Cisco Ccna( 640-802 certification) Cisco Ccnp route( 642-902 certification: implementing Cisco IP routing) Cisco Ccnp switch( 642-813 certification: implementing Cisco IP switched networks) Cisco Ccnp Tshoot( 642-832 certification: Troubleshooting and maintaining Cisco IP networks) experience in Troubleshooting and deploying networks using major L2/ L3 protocols and platforms:- routing protocols( rip, Ospf, Eigrp, Is-Is, Bgp)- data link protocols( Stp, Rstp, Mstp)- first hop redundancy protocols: Hsrp, Vrrp, Glbp- Mpls fundamentals- Cisco, nexus, Juniper, Alcatel SR platforms- Unix/ Linux- basic, master of science( M. S.) degree in" management of networks and services"- Politehnica University- electronics, telecommunications and information technology faculty bachelor of science( B. S) in" telecommunications networks and software"- Politehnica University- electronics, telecommunications and information technology faculty experience in operation and maintenance, planning and deploying large scale networks using high availability/ routing and switching Technologies in the likes of: Hsrp/ Vrrp, Ospf, Bgp, Mpls, LDP experience in working with network automation and deployment tools and scripts using Versioned repositories( git), Jinja Templates, Bash/ python scripts. Cisco certified network professional routing and switching( Ccnp routing and switching),\' Politehnica\' University of Bucharest graduate in 2011, electronics, telecommunications and information technology faculty master in networks and services management at the same faculty, diploma obtained in July 2013. bachelor degree in telecommunications software and networks- routing protocols( rip, Ospf, Eigrp, Is-Is, Bgp)- data link protocols( Stp, Rstp, Mstp, Cdp, Ppp)- first hop redundancy protocols: Hsrp, Vrrp, Glbp- Mpls fundamentals- Cisco, nexus, Juniper, Alcatel SR platforms- Unix/ Linux- basic- good knowledge of Microsoft office suite: word, excel, power point, Visio specialties- Cisco routing and switching- Cisco Ccna certified- Cisco Ccnp route diploma- Sdh/ Dwdm- basic level, Mihnea Tudor. network development engineer- Internet services at Amazon ... bachelor of science( B. S) degree in" telecommunications networks and software" program at Politehnica University- electronics ... Cisco, nexus, Juniper, Alcatel SR platforms ... network engineer at workday ... Linkedin members in Ireland:., senior network analyst at continent 8 technologies, network specialist at Asavie')
# # h = nlp("Linux")
# [(ent.text, ent.label_) for ent in h.ents]

def dictionary_with_title_description(df):
    company_names_description = {}
    
    for i in range(0, len(df)):
        
        doc = nlp(df.loc[i, "summaries"][0])
        
        for X in doc.ents:
            if (X.label_ == 'ORG'):
                
                company_names_description.setdefault(df.loc[i, "id"], [])
                company_names_description[df.loc[i, "id"]].append(X.text)
                
    print("Company names extracted from Summaries : \n",company_names_description)

    return company_names_description
# function call
company_names_description = dictionary_with_title_description(summary)
# adding More Entities and removing duplicates in the company_names_description dictionary
# df_entity = pd.read_csv("entity_other_from_ner.csv") 
# # Converting dumped data into proper format i.e string into list

# # Pass the matched technology column that we got
# def convert_data_format(df,column_name):
    
#     for i in range(0, len(df)):
        
#         if df[column_name][i][0] == '[':
#             res = ast.literal_eval(df[column_name][i])
#             df[column_name][i] = res
            
#         else:
#             x = []
#             x.insert(0, df[column_name][i])
#             df[column_name][i] = x
            
#     return df
# df_entity = convert_data_format(df_entity, "combined_list_unique")
#     k = df_entity[df_entity["id"]=="zPnnWGWHb2gSYnd98GKaag_0000"].index.values
#     list_final = df_entity["combined_list_unique"][k[0]]
#     list_final

# # adding more ents in list
# for key, value in company_names_description.items():
#     k = df_entity[df_entity["id"]==key].index.values
#     list_final = df_entity["combined_list_unique"][k[0]]

#     company_names_description[key] = company_names_description[key]+list_final
company_names_description
# taking unique and removing duplicate names
for key_3, value_3 in company_names_description.items():
    company_names_description[key_3] = list(set(company_names_description[key_3]))
company_names_description["fQ2ZZYOxKpdNYTC6TLFObA_0000"]
# def new_swapped_dict(df, company_names_description):

#     company_names_description_new = {}
    
#     for key, value in company_names_description.items():
#         for j in range(0, len(value)):
#             company_names_description_new[value[j]] = key
    
#     final_dictionary = {}
    
#     for key_2, val in company_names_description_new.items():
        
#         for i in range(0 , len(tech)):
#             b = tech.loc[i, "title"]
#             b=b.lower()
#             key_2 = key_2.lower()
                            
# #             if fuzz.ratio(key_2, b) > 85:
#             if key_2 == b:
#                 final_dictionary.setdefault(val,[])
#                 final_dictionary[val].append(b)
                
#     return company_names_description_new, final_dictionary







# def new_swapped_dict(df, company_names_description):

#     company_names_description_new = {}
    
#     for key, value in company_names_description.items():
#         for l in range(0, len(value)):
#             company_names_description_new[value[l]] = key
    
#     final_dictionary = {}
    
#     for key_2, val in company_names_description_new.items():
        
#         for m in range(0 , len(tech)):
#             b = tech.loc[m, "title"]
#             b=b.lower()
#             key_2 = key_2.lower()
                            
#             if key_2 == b:
#                 final_dictionary.setdefault(val,[])
#                 final_dictionary[val].append(b)
                
#     return company_names_description_new, final_dictionary








def new_swapped_dict(df, company_names_description):

#     company_names_description_new = {}
    
#     for key, value in company_names_description.items():
#         for l in range(0, len(value)):
#             company_names_description_new[value[l]] = key
    
    final_dictionary = {}
    
    for key_2, val_2 in company_names_description.items():
        
        for m in range(0 , len(tech)):
            for o in range(0, len(val_2)):
                b = tech.loc[m, "title"]
                b=b.lower()
                company_names_description[key_2][o] = company_names_description[key_2][o].lower()
                
                if company_names_description[key_2][o] == b:
                    final_dictionary.setdefault(key_2,[])
                    final_dictionary[key_2].append(b)
                
    return final_dictionary
# function call
final_dictionary = new_swapped_dict(summary, company_names_description)
# company_names_description_new
final_dictionary
tech["id"] = tech.index
tech.head()
try:
  
  for k, v in final_dictionary.items():
    # x = summary["summaries"].loc(summary["id"] == k)
    # print(x)
    new_row = {'id':k,'Technology_Names': final_dictionary[k]}
    output = output.append(new_row, ignore_index=True)

except Exception as e :
    print("\n\n", e)
# try:
#   for i in range(0, len(summary)):
#     new_row_2 = {'id':summary["id"][i], 'summaries':summary["summaries"][i], 'Technology_Names':final_dictionary[summary["id"][i]]}
#     output_2 = output.append(new_row, ignore_index=True)

# except Exception as e :
#   print("\n\n", e)
output.head()
# output.to_csv("Dataset/Output/NER-output/NER_train_before_regex_match_lg_truecase.csv", index = False)
len(output)
## Extracting summary to Output sheet
for ind in output.index:
    for i in summary.index:
        
        if output["id"][ind]==summary["id"][i]:
            output["summaries"][ind] = summary["summaries"][i]
len(output)
output.head()
output.to_csv("NER_train_lg_moreTAGS_simple_spacy.csv", index = False)
df_summary = pd.read_csv("NER_train_lg_moreTAGS_simple_spacy.csv")
df_summary.head()
# Lowercasing the summary is important for exact match finding
for ind in output.index:
    output["summaries"][ind][0] = output["summaries"][ind][0].lower()
output.head()
df_summary.head()
# Applying Regex to filter out Invalid Words

def get_exact_match(txt, tech):
    try:
        patt = '|'.join(['\\b'+elem+'\\b' for elem in tech])
        matched_patt = re.findall(patt,txt[0])
    except Exception as e:
        print(e)
        matched_patt = []
    return matched_patt
output['matched_NER_by_borders'] = output.apply(lambda x: get_exact_match(x["summaries"], x["Technology_Names"]), axis = 1)
output.head()
# Applying Regex to filter out Invalid Words

def get_exact_match_2(txt, tech):
    try:
        patt = '|'.join(['\\s'+elem+'\\s' for elem in tech])
        matched_patt = re.findall(patt,txt[0])
    except Exception as e:
        print(e)
        matched_patt = []
    return matched_patt
output['matched_NER_final'] = output.apply(lambda x: get_exact_match_2(x["summaries"], x["matched_NER_by_borders"]), axis = 1)
# output.to_csv("Train_Data/Accuracy_Data/Dataset_to_run_on_all/Output/NER_tagged_data_ner_spacy_own_vocab_100_regex_final.csv", index = False)
# strip by whitespace
for i in range(0,len(output)):
    for j in range(0, len(output['matched_NER_final'][i])):
        output['matched_NER_final'][i][j] = output['matched_NER_final'][i][j].strip()
#     print(output['matched_NER_final'][i])
    
df_company_names_description = pd.DataFrame(list(company_names_description.items()))
df_company_names_description.columns = ["id","NER_Extracted_Tags"]
# df_company_names_description["summaries"] = np.nan
# # Extracting summary to Output sheet
# for ind in df_company_names_description.index:
#     for i in df_summary.index:
        
#         if df_company_names_description["id"][ind]==df_summary["id"][i]:
#             df_company_names_description["summaries"][ind] = df_summary["summaries"][i]
df_company_names_description.head()
# df_company_names_description = df_company_names_description.sort_values(by=['id'], ascending=False)
# df_company_names_description.to_csv("Dataset/Output/NER-output/NER_Extracted_Entities__lg_truecase_moreTAGS.csv", index = False)
output = output.merge(df_company_names_description, 'left')
orig_df = summary.loc[:,["id", "summaries", "Technology"]]
df_final = orig_df.merge(output, 'left', on = ["id"])
df_final.drop(columns = "summaries_y", axis  = 1, inplace = True)
df_final.head()
# # TrueCase summary copy from saved df_summary to df_final summary column for matching id's
# for i in df_final.index:
#     df_final["summaries_x"][i] =df_summary["summaries"].loc[df_summary["id"]==df_final["id"][i]]
for i in df_final.index:
    
    for j in df_summary.index:
        if df_final["id"][i]==df_summary["id"][j]:
            df_final["summaries_x"][i] = df_summary["summaries"][j]
df_final.head()
len(df_final)
df_final["id"].nunique()
df_final.rename(columns = {"summaries_x":"summaries"}, inplace = True)
df_final.head()
final_df = df_final.sort_values(by=['id'], ascending=False)
final_df.head()
final_df["matched_NER_by_borders"] = final_df["matched_NER_by_borders"].fillna(0)
# demo
for i in range(0, len(final_df)):
        if final_df["matched_NER_by_borders"][i] != 0:
            mylist = final_df["matched_NER_by_borders"][i]
            
            mylist = list(dict.fromkeys(mylist))
            final_df["matched_NER_by_borders"][i] = mylist
            
final_df.head()
final_df.to_csv("NER_train_ast_dump_resolve_lg__moreTAGS_tagged_100_simple_spacy.csv", index = False)
final_df.head()
kmp_df = pd.read_csv("KMP_string_Match_newer.csv")
kmp_df.head(2)
type(kmp_df["exact_matched_patt"][1])
# Converting dumped data into proper format i.e string into list

# Pass the matched technology column that we got
def convert_data_format(df,column_name):
    
    for i in range(0, len(df)):
        
        if df[column_name][i][0] == '[':
            res = ast.literal_eval(df[column_name][i])
            df[column_name][i] = res
            
        else:
            x = []
            x.insert(0, df[column_name][i])
            df[column_name][i] = x
            
    return df
kmp_df = convert_data_format(kmp_df, "exact_matched_patt")
type(kmp_df["exact_matched_patt"][0])
kmp_df["exact_matched_patt"][1]
kmp_df["exact_matched_patt"][0]
for i in range(0, len(kmp_df)):
    if len(kmp_df["exact_matched_patt"][i])==0:
        kmp_df["exact_matched_patt"][i] = ["0"]
        
for i in range(0, len(final_df)):
    if (final_df["matched_NER_by_borders"][i])==0:
        final_df["matched_NER_by_borders"][i] = ["0"]
        
type(final_df["matched_NER_by_borders"][0])
for i in range(0,len(kmp_df)):
    print(i)
    if (kmp_df["id"][i]==final_df["id"][i]):
      
            final_df["matched_NER_by_borders"][i] = kmp_df["exact_matched_patt"][i]+final_df["matched_NER_by_borders"][i]            
            final_df["matched_NER_by_borders"][i] = list(set(final_df["matched_NER_by_borders"][i]))
        
#         final_df["matched_NER_by_borders"][i] = (final_df["matched_NER_by_borders"][i]+df_string_match["exact_matched_patt"][i])
#         final_df["matched_NER_by_borders"][i] = list(set(final_df["matched_NER_by_borders"][i]))
final_df
# h = final_df["matched_NER_by_borders"][3].remove(0)
final_df["matched_NER_by_borders"][0][0]
for i in range(0, len(final_df)):
    l = final_df["matched_NER_by_borders"][i]
    final_df["matched_NER_by_borders"][i] = [0 if element == '0' else element for element in l]
# for i in range(0, len(final_df)):
#   l = final_df["matched_NER_by_borders"][i]
#   g = l[:]
#   l.remove(0)
for i in range(0, len(final_df)):
  words = final_df["matched_NER_by_borders"][i]
  stopwords = [0]
  for word in list(words):
    if word in stopwords:
      words.remove(word)
final_df
final_df.to_csv("final_df_string_ner_v2.csv", index = False)<jupyter_output><empty_output><jupyter_text>Accuracy<jupyter_code># fill nan with 0 in dataframe
summary.head()
summary["exact_matched_patt"] = summary["exact_matched_patt"].fillna(0)

# final_df["Technology_Names"] = final_df["Technology_Names"].fillna(0)
# final_df["Technology_Names"] = final_df["Technology_Names"].replace(np.nan, 0)
summary["exact_matched_patt"][2]
def convert_tagged_data_format(df,column_name):
    
    for i in range(0, len(df)):
        print(i)
        
        if df[column_name][i] is not np.nan:
            
            if df[column_name][i][0] == '[':
                pass
                res = ast.literal_eval(df[column_name][i])
                df[column_name][i] = res
                
            else:
                y = []
                y.insert(1, df[column_name][i])
                df[column_name][i] = y
                
        else:
    
                y = []
                y.insert(1, df[column_name][i])
                df[column_name][i] = y
    return df
summary = convert_tagged_data_format(summary, "exact_matched_patt")
summary["exact_matched_patt"][3]
summary.head()
# count of Technology Names in final output list
# importing Counter module 
from collections import Counter 
input_list =  summary["exact_matched_patt"].to_list()
count_input_list = []
input_list[2]
# input_list = pd.Series(input_list, dtype=object).fillna(0).tolist()
len(input_list)
# creating a list with the keys 
for i in input_list:
    if i == 0:
        count_input_list.append(0)
        
    elif i!=0:
        items = Counter(i).keys()
        print("No of unique items in the list are:", len(items))
        count_input_list.append(len(items))
len(count_input_list)
d = pd.DataFrame(count_input_list, columns = ["No of Tech in summary"])
final_df = summary.sort_values(by=['id'], ascending=False)
# fill nan with 0 in dataframe
final_df["exact_matched_patt"] = final_df["exact_matched_patt"].fillna(0)
final_df.head()
final_df.to_csv("Dataset/Output/NER-output/demo_final_df_lg.csv", index = False)
f_Df = pd.read_csv("Dataset/Output/NER-output/demo_final_df_lg.csv")
f_Df.head()
final_count_df = pd.concat([f_Df, d], axis = 1)
final_count_df.head()
final_count_df["No of Tech in summary"][3]
final_count_df["exact_matched_patt"][3]
final_count_df.to_csv("Dataset/Output/NER-output/NER_train_ast_dump_resolve_lg_truecase_tech_count_lg_moreTAGS.csv", index = False)<jupyter_output><empty_output><jupyter_text>** TECH VS COUNT **<jupyter_code>c = pd.read_csv("/content/final_df_string_ner.csv")
c.head(2)
c["matched_NER_by_borders"][0]
# Converting dumped data into proper format i.e string into list

# Pass the matched technology column that we got
def convert_data_format(df,column_name):
    
    for i in range(0, len(df)):
        
        if df[column_name][i][0] == '[':
            res = ast.literal_eval(df[column_name][i])
            df[column_name][i] = res
            
        else:
            x = []
            x.insert(0, df[column_name][i])
            df[column_name][i] = x
            
    return df
c = convert_data_format(c, "matched_NER_by_borders")
# Technology vs COunt and removing duplicates but preserving their counts
df_cm = c
count = []
id_elem = []
elem = []
for ind in df_cm.index:
    d = df_cm["matched_NER_by_borders"][ind]
    for j in range(0, len(d)-1):
        search_elem = df_cm["matched_NER_by_borders"][ind][j]
        i = d.count(search_elem)
        count.append(i)
        elem.append(search_elem)
        id_elem.append(df_cm["id"])
df_final_v2 = pd.DataFrame(list(zip(elem, count)), 
               columns =['Technology_Names', "Count"])
df_final_v2.head()
# Count vs Technology Sheet
save = df_final_v2.groupby(["Technology_Names"]).sum()
save.reset_index(inplace = True)
save = save.sort_values(by=['Count'], ascending=False)
save.head()
len(save)
save.to_csv("Tech_Count_NER_string_dictionary.csv", index = False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup