<jupyter_start><jupyter_text>Get Frameworks and Custom Modules<jupyter_code>import pandas as pd
import numpy as np
print(pd.__version__)
print(np.__version__)
from modules.replace_dep import replace_with_dict, remove_college
replace_with_dict("언홍영")
remove_college("사회과학대학")<jupyter_output><empty_output><jupyter_text>Making Departments dataset<jupyter_code>df_abstact_wrangle = pd.read_csv("data_wrangled/df_abstract_wrangle.csv", encoding="utf-8")
df_abstact_wrangle.sample(5)
departments = list(df_abstact_wrangle["학과"].unique())
print(len(departments), departments[:10])
df_abstact_wrangle["학과"] = df_abstact_wrangle["학과"].fillna(" ")
df_abstact_wrangle["학과"].isnull().values.any()
# 이중전공 두 개로 쪼개기
import re
import numpy as np
from itertools import chain

# return list from series of comma-separated strings
def split_double_majors(str):
    return re.split(',|/', str)

def chainer(series):
    # print(series)
    return list(chain.from_iterable(series.apply(split_double_majors)))

# calculate lengths of splits
# lens = df_abstact_wrangle['학과'].str.split('/').map(len)
lens = df_abstact_wrangle['학과'].apply(split_double_majors).map(len)
lens
# create new dataframe, repeating or chaining as appropriate
res = pd.DataFrame({'대학코드': np.repeat(df_abstact_wrangle['대학코드'], lens),
                    '제목': np.repeat(df_abstact_wrangle['제목'], lens),
                    '학과': chainer(df_abstact_wrangle['학과']),
                    '과정': np.repeat(df_abstact_wrangle['과정'], lens),
                    '년도': np.repeat(df_abstact_wrangle['년도'], lens),
                    'BERT_SCORE': np.repeat(df_abstact_wrangle['BERT_SCORE'], lens),
                    'RNN_SCORE': np.repeat(df_abstact_wrangle['RNN_SCORE'], lens),
                    'REVIEW_LINK': np.repeat(df_abstact_wrangle['REVIEW_LINK'], lens),})
res
df_department = res[["대학코드", "학과", "REVIEW_LINK"]]
df_department.head()<jupyter_output><empty_output><jupyter_text>Preprocessing Department names<jupyter_code># apply preprocessing module
df_department["학과"] = df_department["학과"].apply(lambda x: remove_college(x))
df_department["학과"] = df_department["학과"].apply(lambda x: replace_with_dict(x))
df_department.head()
from collections import Counter
print(Counter(df_department["학과"]))
# print(Counter(temp))
from math import nan

departments = df_department["학과"].unique()
# departments = temp.unique()

korean_deparments = []
english_departments = []
for department in departments:
    if type(department) == float:
        pass
    elif not re.search(r'[a-zA-Z]', department):
        korean_deparments.append(department)
    else:
        english_departments.append(department)

# print(len(korean_deparments), korean_deparments)
print(len(korean_deparments),sorted(korean_deparments))
print(len(english_departments), english_departments)<jupyter_output>178 ['EIC', 'Underwood International College', 'UD 국제학과', 'GSIS', 'EESE', 'Information and Interaction Design', 'Economics (UIC)', 'UIC - PSIR', 'UIC 아시아학과', 'UIC 문학과 문화학', 'UIC', 'TAD (IID & CDM)', 'International Studies', 'UIC경제학과', 'International Trade', 'Finance and Management', '언더우드학과 ud', 'Psychology', 'UIC(Quantitativive Risk Management)', 'UD ECON', '언더우드학부- 정치외교학과 (PSIR)', '국제학부 HASS SDC(지속개발협력)', '언더우드학부-국제학 (UD-IS)', 'UIC 생명과학공학', '국제학부 PSIR', 'Underwood Division', 'PSIR', '동아시아국제학부(EIC)', 'ASD', 'International studies', 'UIC 경제', 'Energy& Environmental Science Engineering', 'UIC ECON', '경영전문대학교 GMBA', 'Economics', '테크노아트학부 CDM', 'UIC국제학과', 'UIC 경제학과', 'UIC 비교문학', 'Social Sciences', 'philosophie', '에너지환경융합(EESE)', 'UIC 생명과학공학과', 'UIC 테크노아트학부', 'Quantitative Risk Management', 'Comparative Literature & Culture', 'UIC PSIR', 'UIC 문화디자인경영', '국제학과 (International Studies)', '계량위험관리(QRM)', 'Underwood International College - Economics', 'UIC - Economics', 'UIC IS', 'UIC Political S[...]<jupyter_text>Yielding metrics for unsupervised clustering(단과대 vs 종합대)<jupyter_code>univ_code = "JP000023"
one_univ_departments = df_department.loc[df_department["대학코드"]==univ_code]["학과"]
one_univ_departments
# 종합대에서 metrics 뽑아낸 예시
from collections import Counter, OrderedDict

dict_departments = Counter(one_univ_departments)
ordered_dict_departments= OrderedDict(dict_departments.most_common())
departments = list(ordered_dict_departments.keys())
number_of_students = list(ordered_dict_departments.values())
print(departments, number_of_students)
# variance of students' departments
np.var(number_of_students)
# 단과대에서 metrics 뽑아낸 예시
from collections import Counter, OrderedDict

univ_code = "DE000003"
one_univ_departments = df_department.loc[df_department["대학코드"]==univ_code]["학과"]
one_univ_departments

dict_departments = Counter(one_univ_departments)
ordered_dict_departments= OrderedDict(dict_departments.most_common())
departments = list(ordered_dict_departments.keys())
number_of_students = list(ordered_dict_departments.values())
print(departments, number_of_students)
# variance of students' departments
np.var(number_of_students)<jupyter_output><empty_output><jupyter_text>Making Dataset for clustering<jupyter_code>def var_and_sum(df_department, univ_code):
    one_univ_departments = df_department.loc[df_department["대학코드"]==univ_code]["학과"]
    dict_departments = Counter(one_univ_departments)
    ordered_dict_departments= OrderedDict(dict_departments.most_common())
    departments = list(ordered_dict_departments.keys())
    number_of_students = list(ordered_dict_departments.values())
    print(np.var(number_of_students), sum(number_of_students), len(departments))
    return np.var(number_of_students), sum(number_of_students), len(departments)
var_and_sum(df_department, univ_code)
univ_list = list(df_department["대학코드"].drop_duplicates().values)[1:]
univ_list[:5]
df_univ = pd.read_csv("data_wrangled/df_univ_wrangled.csv")
df_univ.head(2)
# get univ_name
given_code = "NL000010"
df_univ.loc[df_univ["대학코드"]==given_code]["대학명"].iloc[0]
df_data = pd.DataFrame(columns=["대학코드", "대학명","NUM_VISITS", "NUM_DEP", "VAR"])
for univ_code_i in univ_list:
    univ_name = df_univ.loc[df_univ["대학코드"]==univ_code_i]["대학명"].iloc[0]
    var_data, sum_data, dep_data = var_and_sum(df_department, univ_code_i)
    df_data = df_data.append({"대학코드":univ_code_i, "대학명": univ_name, 
    "NUM_VISITS":sum_data, "NUM_DEP":dep_data, "VAR":var_data}, ignore_index=True)
df_data
df_data.to_csv("./data_wrangled/department_data.csv", index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup