<jupyter_start><jupyter_text>Fixing predict<jupyter_code>import numpy as np
import pandas as pd
import sys

embeddings = pd.read_pickle("embeddings.pkl")
whales = np.load('raw_predictions.npy')

# get array showing for each class where started it's embeddings
ids = embeddings['Id'].values
last_id, starts = ids[0], [0]
for ind, curr_id in enumerate(ids):
    if last_id != curr_id:
        starts.append(ind)
        last_id = curr_id
starts.append(len(ids))
starts = np.array(starts)
embeddings = embeddings.drop(['Id'], axis=1).values
mean_dist = np.empty((whales.shape[0], len(starts)), dtype=float)
mean_dist[:, 0] = sys.maxsize  # new_whale class: constant
#there were computation mistakes before this
mean_emb = np.mean(np.concatenate((embeddings, whales), axis=0), axis = 0)
whales -= mean_emb
embeddings -= mean_emb<jupyter_output><empty_output><jupyter_text>--- predict we want to fix ---<jupyter_code># get 2D array showing mean dist between val embedding and embeddings of group
# using stepped calculation to prevent RAM OOM
class_offset = 1  # to compensate new_whale (class 0) - first column in mean_dist
embeddings_offset = 0
splitted_starts = [starts[:len(starts) // 2 + 1], starts[len(starts) // 2:]]
for starts in [splitted_starts[0]]:
#for starts in splitted_starts:
    starts -= embeddings_offset
    curr_embeddings = embeddings[starts[0]:starts[-1]]

    concat = np.concatenate((curr_embeddings, whales), axis=0)
    
    prod = np.dot(concat, np.transpose(concat))
    sq_norms = np.reshape(np.diag(prod), (-1, 1))

    dist = sq_norms - 2.0 * prod + np.transpose(sq_norms)
    dist = dist[curr_embeddings.shape[0]:, :curr_embeddings.shape[0]]
    dist = np.sqrt(np.maximum(dist, 0.0))
    
    print(curr_embeddings.shape)
    print(concat.shape)
    print(prod.shape)
    print(sq_norms.shape)    
    print(dist.shape)  
    
    i = 0
    print(np.mean(dist[:, starts[i]:starts[i + 1]], axis=1))
    res1 = np.mean(dist[:, starts[i]:starts[i + 1]], axis=1)<jupyter_output>(7885, 128)
(15845, 128)
(15845, 15845)
(15845, 1)
(7960, 7885)
[0.01175699 0.0137426  0.01540905 ... 0.01161361 0.01867516 0.01478532]<jupyter_text>--- same predict but only for class 1 ---<jupyter_code>p = 0
for starts in [np.array([splitted_starts[0][p], splitted_starts[0][p+1]]) ]:
    curr_embeddings = embeddings[starts[0]:starts[-1]]

    concat = np.concatenate((curr_embeddings, whales), axis=0)
    
    prod = np.dot(concat, np.transpose(concat))
    sq_norms = np.reshape(np.diag(prod), (-1, 1))

    dist = sq_norms - 2.0 * prod + np.transpose(sq_norms)
    dist = dist[curr_embeddings.shape[0]:, :curr_embeddings.shape[0]]
    dist = np.sqrt(np.maximum(dist, 0.0))
    
    print(curr_embeddings.shape)
    print(concat.shape)
    print(prod.shape)
    print(sq_norms.shape)    
    print(dist.shape)  
    
    print(np.mean(dist, axis=1))
    res2 = np.mean(dist, axis=1)<jupyter_output>(1, 128)
(7961, 128)
(7961, 7961)
(7961, 1)
(7960, 1)
[0.01175699 0.0137426  0.01540905 ... 0.01161345 0.01867506 0.01478532]<jupyter_text>--- same but with scipy ---<jupyter_code>p = 0
for starts in [np.array([splitted_starts[0][p], splitted_starts[0][p+1]]) ]:
    curr_embeddings = embeddings[starts[0]:starts[-1]]
    res3 = np.linalg.norm(whales - curr_embeddings[0], axis = 1)
res3
res1
np.where((res1 - res2 > 0.00001))[0].shape[0]
np.where((res1 - res3 > 0.00001))[0].shape[0]
from scipy.spatial import distance

dist = distance.cdist(whales, embeddings[:10], 'euclidean')
dist[:, 0]<jupyter_output><empty_output><jupyter_text>It's ok now!  Checking embeddings are collapsed<jupyter_code>import numpy as np
import pandas as pd
import sys

embeddings = pd.read_pickle("embeddings.pkl")
embeddings = embeddings.drop(['Id'], axis=1).values
whales = np.load('raw_predictions.npy')

mean_emb = np.mean(np.concatenate((embeddings, whales), axis=0), axis=0)
mean_emb
np.where(embeddings - mean_emb > 0.05)[0].shape
np.where(whales - mean_emb > 0.05)[0].shape<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup