<jupyter_start><jupyter_text>pre processing all current raw data for action id process    steps:    1 - devide all files by user (with related information)    2 - **merge each user's all related files**    3 - **sort by timestamp**    4 - decide the certain "action key"(exp. 1:login 2:longoff)    5 - break the log into action keys (origional using auto-generated non-repeat num ID with parsed log type)    6 - new line while "logoff" (origional using block_id which not exist here)<jupyter_code>import numpy as np
import pandas as pd
import os
from tqdm import tqdm

LDAP_dir = "./r5.2/LDAP"
Processed_dir = "./pre_processed"
result_dir = "./processed_log"
# get LDAP name list for matching user and email

LDAP_files = []
for _, _, file in os.walk(LDAP_dir):
    LDAP_files.append(file)
LDAP_files = np.array(LDAP_files)
LDAP_files = np.squeeze(LDAP_files)

user_file = pd.read_csv(os.path.join(LDAP_dir, LDAP_files[0]))
print(user_file.shape)
print(user_file.head(5))

for file in LDAP_files:
    temp = pd.read_csv(os.path.join(LDAP_dir, file))
    temp = temp[~temp.user_id.isin(user_file.user_id)]
    user_file = user_file.append(temp)
print(user_file.shape)
def show(file):
    print("shape is ",file.shape)

    print("\nindex and top 5 rows")
    print(file.head(5))
user_file = user_file.drop(columns=["employee_name","role","business_unit","functional_unit","department","team","supervisor"])
show(user_file)
def get_email_by_id(df, id):
    return df[(df['user_id'] == id)].email.to_string().split()[1]

# then test we can use the id to query email address
get_email_by_id(user_file, "SIB0825")<jupyter_output><empty_output><jupyter_text>test for one user<jupyter_code>test_user_id  = "TDG0203"
file_list = []
for _, _, file in os.walk(Processed_dir+'/'+test_user_id):
    file_list.append(file)
file_list
test_email = get_email_by_id(user_file, test_user_id)
test_email
test_email_file = Processed_dir+'/'+test_email+"/email_file.csv"
email_file = pd.read_csv(test_email_file)
show(email_file)
email_file = email_file.drop(columns=["from"])
email_file['action'] = "email"
show(email_file)


logon_file = pd.read_csv(Processed_dir+'/'+test_user_id+'/'+file_list[0][0])
http_file = pd.read_csv(Processed_dir+'/'+test_user_id+'/'+file_list[0][1])
file_file = pd.read_csv(Processed_dir+'/'+test_user_id+'/'+file_list[0][2])
device_file = pd.read_csv(Processed_dir+'/'+test_user_id+'/'+file_list[0][3])
show(logon_file)
logon_file = logon_file.drop(columns='user')
show(logon_file)
logon_file = logon_file.rename(columns={'activity': 'action'})
show(logon_file)
http_file = http_file.drop(columns='user')
show(http_file)
file_file = file_file.drop(columns='user')
show(file_file)
device_file = device_file.drop(columns='user')
show(device_file)
device_file = device_file.rename(columns={'activity': 'action'})
show(device_file)
final_df = logon_file.append(http_file).append(file_file).append(device_file).append(email_file)
show(final_df)
final_df.to_csv("./test_final_raw.csv")<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup