<jupyter_start><jupyter_text>Imports<jupyter_code># Pandas, Numpy and Matplotlib
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
#from graph_tool.all import *<jupyter_output><empty_output><jupyter_text>Get tagged words<jupyter_code># Set name of file
filename = '../data/interim/disease_tags.pkl'

# Read to DataFrame
df = pd.read_pickle(filename)

# Echo
print('Total entries in df: ', df.shape[0])
print('Unique summaries in df: ', df['Id'].nunique())
df.head()<jupyter_output>Total entries in df:  124502
Unique summaries in df:  29417<jupyter_text>Extract titles<jupyter_code>names = pd.DataFrame(df['ont unique_id disease_tag'.split()])
names['id'] = names['ont']+':'+names['unique_id']
names['label']=names['disease_tag']
names.drop(labels='ont unique_id disease_tag'.split(),axis=1, inplace=True)
names.drop_duplicates(subset='id', keep='first',inplace=True)
names.dropna(axis=0,inplace=True)
names.to_csv('titles.csv', index=False)
names.head()<jupyter_output><empty_output><jupyter_text>Clean Data<jupyter_code># Exclude start/end/disease_tag columns
tags = pd.DataFrame(df['Id ont unique_id'.split()])

# Drop NaNs
tags.dropna(axis=0,inplace=True)

# Rename fields, create combined field 'ont:unique_id', create repetition field.
tags['summary_id'] = tags['Id']
tags['disease_id'] = tags['ont']+':'+tags['unique_id']
tags['year'] = 2017 

# Leave only important fields
tags = tags['year summary_id disease_id'.split()]

# Delete duplicates
tags = tags.drop_duplicates()

# Echo
print('Unique summaries in tags: ', tags['summary_id'].nunique())
print('Total tags after cleaning: ', tags.shape[0])
tags.head()<jupyter_output>Unique summaries in tags:  28346
Total tags after cleaning:  61625<jupyter_text>Clean data from summaries with only one tag<jupyter_code># Count tags per summary
tags_by_summary = tags['summary_id disease_id'.split()].groupby('summary_id').count().reset_index()

# Select abstracts with more than one tag
good_summaries = tags_by_summary[tags_by_summary['disease_id']>1]

# Inner Join
clean_tags = pd.merge(tags, good_summaries, on='summary_id')

# Rename columns
clean_tags.rename(columns={'disease_id_x':'disease_id'}, inplace=True)

# Drop extra column
clean_tags = clean_tags['year summary_id disease_id'.split()]

# Echo
clean_tags.head()
%%time 
# About 3 minutes
import itertools
links = list()
for summary in clean_tags['summary_id']:
    tagos = clean_tags[clean_tags['summary_id']==summary]['disease_id']
    links.extend(list(itertools.combinations(tagos,2)))
lynx = pd.DataFrame(links, columns='Source Target'.split())
lynx['Weight'] = 1
lynx = lynx.groupby('Source Target'.split()).count().reset_index()#.sort_values(by='counts',ascending=False)
lynx['Weight'] = lynx['Weight']/lynx['Weight'].max()
lynx.head()
lynx.to_csv('links.csv', index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup