<jupyter_start><jupyter_text>Helper function that displays all the functions that can be invoked on an object.<jupyter_code>def display_object_method(obj):
    object_methods = [method_name for method_name in dir(obj)
                  if callable(getattr(obj, method_name))]
    print(object_methods)
def get_split_words(word_bag):    
    split_words = word_bag.str.split()
    stripped_words = split_words.flatten().map(lambda x: x.strip())
    word_array = stripped_words.filter(lambda x: x!= "")
    return word_array
def get_filtered_words(words):
    stopwords = open("/home/dsp_team_gregory/gregory-p1/data/stopwords.txt", "r").read().split("\n")
    return words.filter(lambda x: x not in stopwords)
def get_words_for_document(document):
    word_bag = db.read_text(document)
    split_words = get_split_words(word_bag)
    long_words = split_words.filter(lambda x: len(x) < 3)
#     words_without_punctuations = long_words.map(puntuation_stripper)
    return get_filtered_words(long_words)    
#     return long_words
def get_IDF(words_by_document):
    unique_words = []
    for words_for_single_document in words_by_document:
        unique_words.append(words_for_single_document.distinct())
    large_bag = db.concat(unique_words)
    frequencies = large_bag.frequencies()
    idf = frequencies.map(lambda x: (x[0], round(math.log((len(words_by_document) + 1)/x[1]), 10)))
    return idf
def get_TF_IDF(tf, idf):
    tf = np.asarray(tf.frequencies().compute())
    tf_idf = {}
    for idf_item in idf: 
        col_name = str(int(idf_item[0], 16))
        tf_idf[col_name] = idf_item[1]
        for i in range(len(tf)):
            tf_item = tf[i]
            if idf_item[0] == tf_item[0]:
                tf_idf[col_name] = float(idf_item[1]) * float(tf_item[1])
                np.delete(tf, i)
                break
    return tf_idf
set_name = "X_small_test.txt"
# set_name = "X_train.txt"
# url = "/home/dsp_team_gregory/gregory-p1/data/train/" + set_name
url = 'https://storage.googleapis.com/uga-dsp/project1/files/' + set_name
file_names = pd.read_csv(url, header=None)[0].to_numpy()
import time

t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
print(current_time)
all_file_words = []
for file_name in file_names:
#     file_name = "/home/dsp_team_gregory/gregory-p1/data/train/" + file_name + ".bytes"
    file_name = "https://storage.googleapis.com/uga-dsp/project1/data/bytes/" + file_name + ".bytes"
    all_file_words.append(get_words_for_document(file_name))
idf = get_IDF(all_file_words)
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
print(current_time)
idf = np.asarray(idf.topk(256, key=0).compute())
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
print(current_time)
cols = np.arange(255, -1, -1).astype(str)
pd_df = pd.DataFrame(columns=cols)
i = 0
for file_words in all_file_words:
    print("working on file number %d" % i)
    i += 1
    tf_idf_bag_array = get_TF_IDF(file_words, idf)
    pd_df = pd_df.append(tf_idf_bag_array, ignore_index=True)
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
print(current_time)
csv_name = "/home/dsp_team_gregory/gregory-p1/data/" + set_name + ".csv"
pd_df.to_csv(csv_name)
csv_name
tf_idf_df = df.from_pandas(pd_df, npartitions=1)
final_dataframe = tf_idf_df.compute()
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
print(current_time)
final_dataframe<jupyter_output><empty_output><jupyter_text>Making the dataframe accessible by the other files<jupyter_code>%store final_dataframe<jupyter_output>Stored 'final_dataframe' (DataFrame)
StarcoderdataJupyterStructuredCleanDedup