<jupyter_start><jupyter_text>Negative sampling Generating some edges which are not present in graph for supervised learning. In other words, we are generating bad links from graph which are not in graph and whose shortest path is greater than 2.<jupyter_code>## This pandas method is super slow, not sure why, compare to csv reader method
# traindf['weight'] = 1
# edges = traindf.set_index(['source_node','destination_node']).T.to_dict('records')[0]

traindf.to_csv('/content/train_woheader.csv', header=False, index=False)
r = csv.reader(open('/content/train_woheader.csv','r'))
edges = dict()
for edge in r:
    edges[(edge[0], edge[1])] = 1
missing_edges = set([])
with tqdm(total=9437519) as pbar:
    while (len(missing_edges)<9437519):
        a=random.randint(1, 1862220)
        b=random.randint(1, 1862220)
        tmp = edges.get((a,b),-1)
        if tmp == -1 and a!=b:
            try:
                if nx.shortest_path_length(g,source=a,target=b) > 2: 
                    missing_edges.add((a,b))
                else:
                    continue  
            except:  
                    missing_edges.add((a,b))              
        else:
            continue
        pbar.update(1)
list(missing_edges)[:10]
# pickle.dump(missing_edges,open('/content/missing_edges_final.p','wb'))<jupyter_output><empty_output><jupyter_text>Train/test split > Tip: We will split positive links and negative links seperatly because we need only positive training data for creating graph and for feature generation.<jupyter_code>#reading total data df
df_pos = traindf.copy()
df_neg = pd.DataFrame(list(missing_edges), columns=['source_node', 'destination_node'])

print("Number of nodes in the graph with edges", df_pos.shape[0])
print("Number of nodes in the graph without edges", df_neg.shape[0])
#Train test split 
#Spiltted data into 80-20
X_train_pos, X_test_pos, y_train_pos, y_test_pos  = train_test_split(df_pos,np.ones(len(df_pos)),test_size=0.2, random_state=9)
X_train_neg, X_test_neg, y_train_neg, y_test_neg  = train_test_split(df_neg,np.zeros(len(df_neg)),test_size=0.2, random_state=9)

print('='*60)
print("Number of nodes in the train data graph with edges", X_train_pos.shape[0],"=",y_train_pos.shape[0])
print("Number of nodes in the train data graph without edges", X_train_neg.shape[0],"=", y_train_neg.shape[0])
print('='*60)
print("Number of nodes in the test data graph with edges", X_test_pos.shape[0],"=",y_test_pos.shape[0])
print("Number of nodes in the test data graph without edges", X_test_neg.shape[0],"=",y_test_neg.shape[0])
# X_train = X_train_pos.append(X_train_neg,ignore_index=True)
# y_train = np.concatenate((y_train_pos,y_train_neg))
# X_test = X_test_pos.append(X_test_neg,ignore_index=True)
# y_test = np.concatenate((y_test_pos,y_test_neg)) 

# X_train.to_csv('train_after_eda.csv',header=False,index=False)
# X_test.to_csv('test_after_eda.csv',header=False,index=False)
# pd.DataFrame(y_train.astype(int)).to_csv('train_y.csv',header=False,index=False)
# pd.DataFrame(y_test.astype(int)).to_csv('test_y.csv',header=False,index=False)
# #removing header and saving
# X_train_pos.to_csv('train_pos_after_eda.csv',header=False, index=False)
# X_test_pos.to_csv('test_pos_after_eda.csv',header=False, index=False)
# X_train_neg.to_csv('train_neg_after_eda.csv',header=False, index=False)
# X_test_neg.to_csv('test_neg_after_eda.csv',header=False, index=False)

data_path_silver = './data/silver'
if not os.path.exists(data_path_silver):
    os.makedirs(data_path_silver)

def store_df(df, name):
    df.to_parquet(os.path.join(data_path_silver,name+'.parquet.gzip'), compression='gzip')

# store_df(X_train_pos, 'X_train_pos')
# store_df(X_test_pos, 'X_test_pos')
# store_df(X_train_neg, 'X_train_neg')
# store_df(X_test_neg, 'X_test_neg')
# store_df(X_train, 'X_train')
# store_df(X_test, 'X_test')
store_df(pd.DataFrame(y_train.astype(int), columns=['weight']), 'y_train')
store_df(pd.DataFrame(y_test.astype(int), columns=['weight']), 'y_test')
train_graph = nx.from_pandas_edgelist(X_train_pos,
                            source='source_node',
                            target='destination_node',
                            create_using=nx.DiGraph())

test_graph = nx.from_pandas_edgelist(X_test_pos,
                            source='source_node',
                            target='destination_node',
                            create_using=nx.DiGraph())
print(nx.info(train_graph))
print(nx.info(test_graph))
# finding the unique nodes in both train and test graphs
train_nodes_pos = set(train_graph.nodes())
test_nodes_pos = set(test_graph.nodes())

trY_teY = len(train_nodes_pos.intersection(test_nodes_pos))
trY_teN = len(train_nodes_pos - test_nodes_pos)
teY_trN = len(test_nodes_pos - train_nodes_pos)

print('no of people common in train and test -- ',trY_teY)
print('no of people present in train but not present in test -- ',trY_teN)
print('no of people present in test but not present in train -- ',teY_trN)
print(' % of people not there in Train but exist in Test in total Test data are {} %'.format(teY_trN/len(test_nodes_pos)*100))
!git status
!git add .
!git commit -m 'added silver data layer'
!git push origin main<jupyter_output>[main 9ebae1e] added silver data layer
 7 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 data/silver/X_test_neg.parquet.gzip
 create mode 100644 data/silver/X_test_pos.parquet.gzip
 create mode 100644 data/silver/X_train_neg.parquet.gzip
 create mode 100644 data/silver/X_train_pos.parquet.gzip
 create mode 100644 data/silver/y_test.parquet.gzip
 create mode 100644 data/silver/y_train.parquet.gzip
 create mode 100644 graph_sample.pdf
Counting objects: 11, done.
Delta compression using up to 2 threads.
Compressing objects: 100% (11/11), done.
Writing objects: 100% (11/11), 199.24 MiB | 6.83 MiB/s, done.
Total 11 (delta 2), reused 0 (delta 0)
remote: Resolving deltas: 100% (2/2), completed with 1 local object.[K
remote: warning: GH001: Large files detected. You may want to try Git Large File Storage - https://git-lfs.github.com.[K
remote: warning: See http://git.io/iEPt8g for more information.[K
remote: warning: File data/silver/X_train_neg.parquet.gzip is 79.90 MB;[...]
StarcoderdataJupyterStructuredCleanDedup