<jupyter_start><jupyter_text>train_kmeans.py<jupyter_code># python2 scripts/train_kmeans.py select.mfcc.csv $cluster_num kmeans.${cluster_num}.model || exit 1;
sys.argv = ['scripts/train_kmeans.py', 'select.mfcc.csv', '100', 'kmeans.100.model']
# python2 scripts/train_kmeans.py select.mfcc.csv 10 kmeans.10.model || exit 1;
if __name__ == '__main__':
    if len(sys.argv) != 4:
        print "Usage: {0} mfcc_csv_file cluster_num output_file".format(sys.argv[0])
        print "mfcc_csv_file -- path to the mfcc csv file"
        print "cluster_num -- number of cluster"
        print "output_file -- path to save the k-means model"
        exit(1)

    mfcc_csv_file = sys.argv[1]; output_file = sys.argv[3]
    cluster_num = int(sys.argv[2])
    
    train_sample = pd.read_csv(mfcc_csv_file, header=None, sep=';')
    start_time = time.time()
    n_clusters = cluster_num
    n_init = 5
    model = KMeans(n_clusters = n_clusters, random_state = 0, n_init = n_init, n_jobs = -1).fit(train_sample)
    
    filename = output_file+'.sav'
    pickle.dump(model, open(filename, 'wb'))
    print "===== The time consuming of Kmeans clustering : {} seconds =====".format((time.time() - start_time))
#     print mfcc_csv_file
#     print output_file
#     print cluster_num
    print "K-means trained successfully!"

mfcc_csv_file = sys.argv[1]; output_file = sys.argv[3]
cluster_num = int(sys.argv[2])
train_sample = pd.read_csv(mfcc_csv_file, header=None, sep=';')
start_time = time.time()
n_clusters = cluster_num
n_init = 5
output_file = KMeans(n_clusters = n_clusters, random_state = 0, n_init = n_init, n_jobs = -1).fit(train_sample)
print("===== The time consuming of Kmeans clustering : {} seconds =====".format((time.time() - start_time)))<jupyter_output><empty_output><jupyter_text>create_kmeans.py<jupyter_code># python2 scripts/create_kmeans.py kmeans.${cluster_num}.model $cluster_num list/all.video || exit 1;
sys.argv = ['scripts/create_kmeans.py', 'kmeans.3.model', '3', 'list/all.video']
#!/bin/python
import numpy
import os
import cPickle
from sklearn.cluster.k_means_ import KMeans
import sys
# Generate k-means features for videos; each video is represented by a single vector

if __name__ == '__main__':
    if len(sys.argv) != 4:
        print "Usage: {0} kmeans_model, cluster_num, file_list".format(sys.argv[0])
        print "kmeans_model -- path to the kmeans model"
        print "cluster_num -- number of cluster"
        print "file_list -- the list of videos"
        exit(1)

    kmeans_model = './'+sys.argv[1]+'.sav'; file_list = sys.argv[3]
    cluster_num = int(sys.argv[2])

    # load the kmeans model
    #kmeans = pickle.load(open(kmeans_model,"rb"))
#     model = pickle.load(open(filename, 'rb'))
    
    print kmeans_model
    print file_list
    print cluster_num
    print "K-means features generated successfully!"

if len(sys.argv) != 4:
    print "Usage: {0} kmeans_model, cluster_num, file_list".format(sys.argv[0])
    print "kmeans_model -- path to the kmeans model"
    print "cluster_num -- number of cluster"
    print "file_list -- the list of videos"
    exit(1)

kmeans_model = './'+sys.argv[1]+'.sav'; file_list = sys.argv[3]
cluster_num = int(sys.argv[2])

# load the kmeans model
kmeans_model = './hw1_git/11775-hws/hw1_code/{}.sav'.format(sys.argv[1])
kmeans = pickle.load(open(kmeans_model,"rb"))
model = kmeans
# path = './hw1_git/11775-hws/videos/*.mp4'
path = './mfcc/*.csv'

filelist = []

for file in glob.glob(path):
    filelist.append(file)
filelist
def get_features(k, model, path_list):
    loaded_model= model
    start_time = time.time()
    features_dict = dict()
    filelist = path_list
    for i in range(len(filelist)):
#     for i in range(10):        
        if i % 1000 == 0: 
            print('{}th step progressing....'.format(i)) 
        else: 
            pass
        data = pd.read_csv(filelist[i], sep = ';', header = None)
        pred_centers = loaded_model.predict(data)
        num_clusters = k
        bow_preds = np.zeros((1, num_clusters))

        for ind in pred_centers:
            bow_preds[0, ind] += 1
        norm_feat = (1.0 * bow_preds)/np.sum(bow_preds)
        features_dict[i] = pd.DataFrame(norm_feat)

    features_total = features_dict[0].copy()
    for i in range(1, len(features_dict)):
        foo = features_dict[i].copy()
        features_total = pd.concat([features_total, foo], axis = 0)
        features_total = features_total.reset_index().drop('index', axis = 1)
        
    print("===== The time consuming of getting features : {} seconds =====".format((time.time() - start_time)))
    return features_total
total_features = get_features(cluster_num, model = model, path_list = filelist)
total_features.to_csv('./hw1_git/11775-hws/hw1_code/total_features_k{}.csv'.format(cluster_num), index=False)<jupyter_output><empty_output><jupyter_text>create_asrfeat.py<jupyter_code>#python2 scripts/create_asrfeat.py vocab list/all.video || exit 1;
if __name__ == '__main__':
    if len(sys.argv) != 3:
        print "Usage: {0} vocab_file, file_list".format(sys.argv[0])
        print "vocab_file -- path to the vocabulary file"
        print "file_list -- the list of videos"
        exit(1)


    print "ASR features generated successfully!"
# path = './hw1_git/11775-hws/videos/*.mp4'
path = '../asrs/*.txt'

filelist = []

for file in glob.glob(path):
    filelist.append(file)
filelist
def concatenate_list_data(list):
    result= ''
    for element in list:
        result += str(element)
    return result
text = []
for i in range(len(filelist)):
# for i in range(10):
    with open (filelist[i], "r") as myfile:
        data=myfile.readlines()
        data=concatenate_list_data(data)
    text.append(data)
    if i % 1000 == 0:
        print('{}th txt file reading...'.format(i))
    else: pass
vect = CountVectorizer(stop_words="english")
# vect = CountVectorizer()
bow = vect.fit_transform(text).toarray()
norm_bow = normalize(bow, norm = 'l1', axis=1)
norm_data = pd.DataFrame(norm_bow)
norm_data.to_csv('./hw1_git/11775-hws/hw1_code/total_astfeat.csv', index = False)<jupyter_output><empty_output><jupyter_text>create_kmeas.py & create_asrfeat.py 이어서<jupyter_code>cluster_num = 100
total_features = pd.read_csv('./total_features_k{}.csv'.format(cluster_num))
total_features.head(3)
filelist
i=0
match_front = re.search('mfcc/', filelist[i])
match_end = re.search('.mfcc.csv', filelist[i])
match_front.end()
match_end.start()
filelist[i]
filelist[i][match_front.end()]
# path = './hw1_git/11775-hws/videos/*.mp4'
path = './mfcc/*.csv'

filelist = []

for file in glob.glob(path):
    filelist.append(file)
# Making Video name
video_name_ind = []
for i in range(len(filelist)):
    import re
    match_front = re.search('mfcc/', filelist[i])
    match_end = re.search('.mfcc.csv', filelist[i])
    video_name_ind.append(filelist[i][match_front.end():match_end.start()])
    video_name = pd.DataFrame({'video': video_name_ind})
# Making features columns
k = cluster_num
column_name = ['video']
for i in range(k):
    column_name.append('feature_{}'.format(i))

total_data = pd.concat([video_name, total_features], axis = 1)
total_data.columns = column_name
#os.chdir('./hw1_git/11775-hws/hw1_code/')
train_ind = pd.read_csv('./list/train', sep = ' ', header = None)
valid_ind = pd.read_csv('./list/val', sep = ' ', header = None)
test_ind = pd.read_csv('./list/test.video', sep = ' ', header = None)

train_ind['Data'] = 'TRAIN'
valid_ind['Data'] = 'VALID'
test_ind[1] = 'UNK'
test_ind['Data'] = 'TEST'

train_ind.columns = ['video','target','Data']
valid_ind.columns = ['video','target','Data']
test_ind.columns = ['video','target','Data']
data_lable = pd.concat([train_ind, valid_ind, test_ind], axis = 0).reset_index().drop('index', axis = 1)
# data_lable['target_p001'] = 
data_lable['target_p001'] = data_lable['target']
data_lable['target_p002'] = data_lable['target']
data_lable['target_p003'] = data_lable['target']
data_lable['target_p001_10'] = 1
data_lable['target_p002_10'] = 1
data_lable['target_p003_10'] = 1

data_lable['target_p001'][data_lable['target'] != 'P001'] = 'Other'
data_lable['target_p002'][data_lable['target'] != 'P002'] = 'Other'
data_lable['target_p003'][data_lable['target'] != 'P003'] = 'Other'
data_lable['target_p001_10'][data_lable['target'] != 'P001'] = 0
data_lable['target_p002_10'][data_lable['target'] != 'P002'] = 0
data_lable['target_p003_10'][data_lable['target'] != 'P003'] = 0
total_mart = total_data.merge(data_lable, how = 'right', on = 'video')
total_mart = total_mart.fillna(0)
train_mart = total_mart[total_mart['Data'] == 'TRAIN']
valid_mart = total_mart[total_mart['Data'] == 'VALID']
test_mart  = total_mart[total_mart['Data'] == 'TEST']
# total_mart.to_csv('./datamart_total_k{}.csv'.format(cluster_num), index=False)
# train_mart.to_csv('./datamart_train_k{}.csv'.format(cluster_num), index=False)
# valid_mart.to_csv('./datamart_valid_k{}.csv'.format(cluster_num), index=False)
# test_mart.to_csv('./datamart_test_k{}.csv'.format(cluster_num), index=False)
print(train_mart.shape, valid_mart.shape, test_mart.shape)
def modeling_ap_SVM(k, train_data, valid_data, target = 'target_p001_10'):
    start_time = time.time()
    k = k
    train_mart = train_data
    valid_mart = valid_data
    target = target
    
    X_train = train_mart.iloc[:,1:k+1]
    y_train = train_mart[target]
    X_valid = valid_mart.iloc[:,1:k+1]
    y_valid = valid_mart[target]
    
    model = SVC(kernel=chi2_kernel, probability=True)
    model.fit(X_train, y_train)
    y_preds = model.predict(X_valid)
    y_probs = model.predict_proba(X_valid)
    results = average_precision_score(y_true=y_valid.values, y_score=y_probs[:,1])
    print("===== The time consuming of SVM Modeling : {} seconds =====".format((time.time() - start_time)))   
    print(results)
    return results

def modeling_ap_AdaB(k, train_data, valid_data, target = 'target_p001_10'):
    start_time = time.time()
    k = k
    train_mart = train_data
    valid_mart = valid_data
    target = target
    
    X_train = train_mart.iloc[:,1:k+1]
    y_train = train_mart[target]
    X_valid = valid_mart.iloc[:,1:k+1]
    y_valid = valid_mart[target]
    
    model = AdaBoostClassifier(n_estimators=200, random_state=0)
    model.fit(X_train, y_train)
    y_preds = model.predict(X_valid)
    y_probs = model.predict_proba(X_valid)
    results = average_precision_score(y_true=y_valid.values, y_score=y_probs[:,1])
    print("===== The time consuming of AdaBoosting Modeling : {} seconds =====".format((time.time() - start_time)))   
    print(results)
    return results

def modeling_ap_Boost(k, train_data, valid_data, target = 'target_p001_10'):
    start_time = time.time()
    k = k
    train_mart = train_data
    valid_mart = valid_data
    target = target
    
    X_train = train_mart.iloc[:,1:k+1]
    y_train = train_mart[target]
    X_valid = valid_mart.iloc[:,1:k+1]
    y_valid = valid_mart[target]
    
    model = GradientBoostingClassifier(n_estimators=200, random_state=0)
    model.fit(X_train, y_train)
    y_preds = model.predict(X_valid)
    y_probs = model.predict_proba(X_valid)
    results = average_precision_score(y_true=y_valid.values, y_score=y_probs[:,1])
    print("===== The time consuming of Boosting Modeling : {} seconds =====".format((time.time() - start_time)))   
    print(results)
    return results

def modeling_ap_xgb(k, train_data, valid_data, target = 'target_p001_10'):
    start_time = time.time()
    k = k
    train_mart = train_data
    valid_mart = valid_data
    target = target
    
    X_train = train_mart.iloc[:,1:k+1]
    y_train = train_mart[target]
    X_valid = valid_mart.iloc[:,1:k+1]
    y_valid = valid_mart[target]
    
    model = XGBClassifier()
    model.fit(X_train, y_train)
    y_preds = model.predict(X_valid)
    y_probs = model.predict_proba(X_valid)
    results = average_precision_score(y_true=y_valid.values, y_score=y_probs[:,1])
    print("===== The time consuming of XgBoosting Modeling : {} seconds =====".format((time.time() - start_time)))   
    print(results)
    return results
AdaB_results_p001 = modeling_ap_AdaB(k=100, train_data = train_mart, valid_data = valid_mart, target = 'target_p001_10')
Xgb_results_p002 = modeling_ap_xgb(k=100, train_data = train_mart, valid_data = valid_mart, target = 'target_p002_10')
Xgb_results_p003 = modeling_ap_xgb(k=100, train_data = train_mart, valid_data = valid_mart, target = 'target_p003_10')<jupyter_output>===== The time consuming of AdaBoosting Modeling : 1.33247709274 seconds =====
0.227125087009372
===== The time consuming of XgBoosting Modeling : 0.334847927094 seconds =====
0.4977064036903855
===== The time consuming of XgBoosting Modeling : 0.362941980362 seconds =====
0.30549811972695673
StarcoderdataJupyterStructuredCleanDedup