<jupyter_start><jupyter_text>notes on that, it took me 3 hours just to create a file with 13k line of rows and i calculated that it would never able to finish the process for 2 billion images<jupyter_code># more_stuff = [img_features.reshape(1,-1), img_features.reshape(1,-1)]
# # test_cos = create_cosine_table(['a'], more_stuff)
# # print(test_cos)
# cosine_similarity(more_stuff)
# # print(img_features.reshape(1,-1))


df = pd.read_csv('cos_sim_table_3.txt', sep=" ", header=None)
# df.columns = ["image_i_id", 'image_j_id', 'cosine']
df.head()
csv_path = 'styles.csv'
style_df = pd.read_csv(csv_path)
style_df.head()

all_article_hist = {}
for pro_type in style_df['articleType']:
    if pro_type in all_article_hist:
        all_article_hist[pro_type] += 1
    else:
        all_article_hist[pro_type] = 1
# print(all_article_hist)
# print(sorted(all_article_hist, reverse=False))      
# style_df['articleType'].value_counts()
# there are 143 different types of products
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
def get_cosine_similarity(v1, v2):
    result = cosine_similarity(v1, v2)
    return result
# a = [[2, 0, 1, 1, 0, 2, 1, 1]]

# b = [[2, 1, 1, 0, 1, 1, 1, 1]]
# img_features = img_features.reshape(1, 7*7*512)
# # print("yhat shape->", yhat.shape)

# # print(get_cosine_similarity(a, a))
# print("yhat cos sim: ")
# # print(img_features)
# print(get_cosine_similarity(img_features, img_features))
def preprocess_image_for_vgg(filepath, img_id):
    """
    Takes an image and returns 4D array of features from vgg16 model
    
    Args:
        image_path: path to image
    """
    # load an image from file
    original_img = load_img(filepath + img_id, target_size=(224, 224))
    # convert the image pixels to a numpy array
    nparry_img = img_to_array(original_img)
    # # prepare the image for the VGG model
    image = np.expand_dims(nparry_img, axis=0)
    # print("image shape:", image.shape)
    processed_image = preprocess_input(image)
#     print("preprocess")
#     print(processed_image)
    # predict the probability across all output classes
    
    return processed_image


def get_feature_vector_from_vgg(processed_image):
    # dont use include top for vgg, because we dont want mlp part
    model = VGG16(weights='imagenet', include_top=False)
    
    feature_vector = model.predict(processed_image)
    return feature_vector


a = [[2, 0, 1, 1, 0, 2, 1, 1]]
b = [[2, 1, 1, 0, 1, 1, 1, 1]]
c = [[2, 3, 6, 1, 0, 2, 8, 0]]
test_for_cos_df = a + b + c
print(test_for_cos_df)
def get_all_image_ids(filepath):
    images_ids = []
    # listing all the image names into list they are ids of the image
    for img in os.listdir(filepath):
        if img.endswith('jpg'):
            images_ids.append(img)
    
    return images_ids

# reshape the feature_vector so we can find cosine similarity score
def reshape_feature_vector(feature_vector):
    """
    Takes 4d array, return 2D representation
    """
    # this is the vgg output 7, 7, 512
    return feature_vector.reshape(1, 7*7*512)
    
# print(reshape_feature_vector(img_features))
# make vgg_feature list 
def list_of_vgg_features(all_imgs):
    # creates the list of vgg features and reshape it ready for making cosine sim score later
    all_feature_vectors = []
    
    for index, img in enumerate(all_imgs):
        feature_vector = get_feature_vector_from_vgg(img)
        # reshape the feature vector
        feature_vector = reshape_feature_vector(feature_vector)
        
        all_feature_vectors.append(feature_vector)
        
    # return all feature vectors
    return all_imgs, all_feature_vectors
        
# def get_cosine_similarity(v1, v2):
#     result = cosine_similarity(v1, v2)
#     return result
def create_cosine_table(list_of_ids, list_of_features):
    """Creates a cosine similarity table from list of vectors"""
    cos_sims = cosine_similarity(list_of_features)
    print(cos_sims)
    cos_table = pd.DataFrame(cos_sims, columns=list_of_ids, index=list_of_ids)
    # turn the cos sim into dataframe
    return cos_table
def get_cosine_table(list_vector):
    cos_list = []
    for i in range(len(list_vector)):
        x = np.array(list_vector[i]).reshape(1, -1)
        for j in range(len(list_vector)):
            y = np.array(list_vector[j]).reshape(1, -1)
            score = cosine_similarity(x, y)
            cos_list.append(score)
    
    df_col = ["a", "b", "c"]
    idx = ["a", "b", "c"]
    print(np.array(cos_list))
    cos_df = pd.DataFrame(cos_list, columns=df_col, index=idx)
    cos_df.head()
    print(cos_list)
    


# #     cos_df.sort_values(by=df_col, axis=1, ascending=False, inplace=False, kind='quicksort', na_position='last')
#     return cos_df
# test_cos_table(test_for_cos_df)
# test_cos_table = test_cos_table(test_for_cos_df)
# test_cos_table
def get_k_most_similar_to(df, given_img, k_similar=5):
    # this gets the rows
    k_images = df[given_img].sort_values(ascending=False)[1:k_similar+1].index
    # this get the corresod=nding scores
    k_images_score = df[given_img].sort_values(ascending=False)[1:k_similar+1]
    for img in range(len(k_images)):
        original = k_images[i]
        plt.imshow(original)
        plt.show()
        print("Similarity score:", k_images_score[i])
def generate_vectors(iterable):
    for item in iterable:
        yield item
# filepath="../fashion-product-images-small/images/"
# list_images = get_all_image_ids(filepath)

# all_image_generator = generate_vectors(list_images)
# all_processed_images = []
# for i, image in enumerate(all_image_generator):
#     processed_image = preprocess_image_for_vgg(filepath, image)
#     if i == 40000:
#         print(processed_image)
#     all_processed_images.append(processed_image)
        
# # print(all_processed_images)
# len(all_processed_images)
# # now make a vgg vectors
# all_vgg_features = []
# vgg_generator = generate_vectors(all_processed_images)
# for image in vgg_generator:
#     feature_vector = get_feature_vector_from_vgg(image)
#     # reshape it before we put into feature vector
#     feature_vector = reshape_feature_vector(feature_vector)
#     # now append it to all features list
#     all_vgg_features.append(feature_vector.flatten())
    
len(all_vgg_features)
def run_all(filepath):
    print("======================STARTING============================")
    all_images = get_all_image_ids(filepath)
    print("some examples: ", all_images[:20])
    # we preprocess the image to make it ready to pass to vgg model
    print("prepocess")
    all_processed_images = []
#     all_images = np.random.choice(all_images, 5)
#     all_images = all_images[:5]
#     gen = generate_vectors(all_images)
    for image in all_images:
        processed_image = preprocess_image_for_vgg(filepath, image)
        all_processed_images.append(processed_image)
        
    print("Now we have preprocessed image for vgg")
    all_feature_vectors = []
    for image in all_processed_images:
        # create feature vector
        feature_vector = get_feature_vector_from_vgg(image)
#         print("making vgg")
        # reshape it before we put into 
#         print("feature shape:", feature_vector.shape)
        feature_vector = reshape_feature_vector(feature_vector)
#         print("after reshaping", feature_vector)
        
        # now append it to all features list
        all_feature_vectors.append(feature_vector.flatten())
    
    print("Now we have all feature vector!")
    all_feature_vectors = np.array(all_feature_vectors)
    print("shape of the feature vector")
    print(all_feature_vectors.shape)
#     create cos sim table

    
    print("we have list of features ready to pass cosine similarity!")
    print("passing...")
    cos_df = create_cosine_table(all_images, all_feature_vectors)
    if len(cos_df) < 2000:
        cos_df.to_csv('cos_similarity_small_table.csv', sep='\t', encoding='utf-8')
    else:
        cos_df.to_csv('cos_similarity_table.csv', sep='\t', encoding='utf-8')
    return cos_df
#     return all_feature_vectors
# filepath = "../fashion-product-images-small/images/"
# result = run_all(filepath)
# # print(result.shape)

result['9733.jpg'].sort_values(ascending=False)[:3].index
result['9733.jpg'].sort_values(ascending=False)[:3]
result
# all_images = get_all_image_ids(filepath)
# smaller_chunk_of_images = np.random.choice(all_images, 2000)
# with open('smaller_chunk_of_samples.txt', 'a') as file:
#     for img_id in smaller_chunk_of_images:
#         file.write(img_id+'\n')
# import sys
# import numpy
# numpy.set_printoptions(threshold=sys.maxsize)

# (smaller_chunk_of_images)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup