<jupyter_start><jupyter_text>**크롤링** **1 크롤링 수집함수 생성하기**<jupyter_code># Web Crawler 함수
from urllib import parse
from urllib import parse, request

def crawler_naver(url):
    browser = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
    return request.urlopen(request.Request(url, headers = {'User-Agent':browser})).read().decode('utf8')
# 네이버 Shopping 수집결과
from lxml.html import fromstring, tostring
def navershop_finder(token_search):
    query = {
        'query' : token_search,
        'frm' : 'NVSHATC',
    } 
    url_query = parse.urlencode(query, encoding='UTF-8', doseq=True)
    resp = crawler_naver("https://search.shopping.naver.com/search/all.nhn?" + url_query)
    resp_lxml = fromstring(resp)
    try:
        resp_lxml = resp_lxml.xpath('//*[@class="list_basis"]')[0] # 검색결과 목록
        resp_lxml = resp_lxml.xpath('.//li')                       # 개별 list
        result = []
        for _ in resp_lxml:
            category = _.xpath('.//div[@class="basicList_depth__2QIie"]//text()')
            if category:
                result.append(">".join(category))
        return result
    except:
        pass<jupyter_output><empty_output><jupyter_text>**네이버 쇼핑 Categoty 수집기** **1 네이버 쇼핑 카테고리 크롤링**제품의 목록단위 정보 수집하기<jupyter_code># result = {}
# from tqdm import tqdm
# import time
# from lxml.html import fromstring, tostring
# for number in tqdm(range(11)):
#     url = f"https://search.shopping.naver.com/category/category/500000{number:02}" # (0 ~ 10)
#     resp = crawler_naver(url)
#     resp_lxml_raw = fromstring(resp)
#     text_key = resp_lxml_raw.xpath('//h2[contains(@class, "category_tit")]//text()')[0]
#     result[text_key] = [_ for _ in resp_lxml_raw.xpath('//h3//text()') if _ != "보기"]
#     time.sleep(1)

# import pandas as pd
# item_category = pd.DataFrame(dict([(k,pd.Series(v)) for k,v in result.items() ])).fillna('')
# # item_category.T.to_csv('categoty_.csv', index=None)
# item_category.to_csv('categoty_naver.csv', index=None)<jupyter_output><empty_output><jupyter_text>**2 크롤링 결과값 활용하기**저장된 데이터를 활용하여 내용 분석하기<jupyter_code>import pandas as pd
dict_category = pd.read_csv('./data/categoty_naver.csv').to_dict(orient='list')
dict_category = {k: [_ for _ in items if str(_) != 'nan']   for k, items in dict_category.items()}
print(", ".join(list(dict_category.keys())))

# dict 제품 구분에서 item 내용만 추출하기
item_values = []
for _, v in dict_category.items():
    item_values += v
item_values = sorted(set(item_values))
print(f"total_length : {len(item_values)} 개")<jupyter_output>패션의류, 패션잡화, 화장품/미용, 디지털/가전, 가구/인테리어, 출산/육아, 식품, 스포츠/레저, 생활/건강, 여가/생활편의, 면세점
total_length : 204 개<jupyter_text>**Item 내용을 카테고리에 적용하기**  **1 마을로 아이템 목록 불러오기**수집 데이터 불러오기<jupyter_code>%whos | grep list & str
import pandas as pd
items = pd.read_csv('./data/maulo_items.csv', encoding="cp949")["Title"].to_list()
items_set = []
for _ in items:
    _ = _.split('/')[0]
    if _ not in items_set:
        items_set.append(_)
len(items), len(items_set)<jupyter_output><empty_output><jupyter_text>**2 네이버 쇼핑 검색**- **[각 품종별 계량단위](http://blog.daum.net/gun515/15507743)** - **[품종별 단위](http://blog.daum.net/_blog/BlogTypeView.do?blogid=0HlvU&articleno=8373563&_bloghome_menu=recenttext)**<jupyter_code># Naver Shopping Get
def naver_finder(items_set):
    r"""Naver Shopping 에서 Item Category 결과추출함수"""
    import numpy as np
    from random import randint
    from time import sleep
    from tqdm import tqdm    
    result, errors = {}, []
    for token_search in tqdm(items_set):
        sleep(0.1 * randint(5,10))
        token_results = navershop_finder(token_search)

        # 검색 결과가 있는경우
        if token_results:
            result_count  = {}
            for _ in token_results:
                if _ not in list(result_count.keys()): 
                    result_count[_] = 0
                else: 
                    result_count[_] +=1

            # 검색결과 1개의 분류만 있는경우
            if len(list(result_count.keys())) == 1:
                result[token_search] = list(result_count.keys())[0]
            # 검색결과 2개만 발견된 경우 (짧은 분류기준 활용하기)
            elif len(list(result_count.keys())) == 2:
                list_temp = list(result_count.keys())
                result[token_search] = list_temp[np.argmin([len(_) for _ in list_temp])]            
            # 검색결과 여러개 발견시 갯수많은 1개 저장
            else:
                for _, v in result_count.items():
                    if v == max(result_count.values()):
                        result[token_search] = _
        else: 
            # 검색 결과가 없는 경우
            result[token_search] = ''
            errors.append(token_search)
    print('Error counts is {}. working is Done...'.format(len(errors)))
    return result

# Running the Finder Function
# naver_finder(items_set[:1])<jupyter_output><empty_output><jupyter_text>**3 결과 데이터 저장 및 활용하기**[pickle 과 zip 을 활용한 저장](https://wikidocs.net/8929)<jupyter_code>import pickle, gzip
# with gzip.open('./data/mauloItemNaver.pickle', 'wb') as f:
# with open('./data/mauloItemNaver.pickle', 'wb') as f:
#     pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
# print("saving is done.")
# import pickle, gzip
# with open('./data/mauloItemNaver.pickle', 'rb') as f:
#     items_maulo_category = pickle.load(f)
# print("loading the date.")<jupyter_output><empty_output><jupyter_text>**추가 작업의 진행** **1 Noun Tokenizer**식품 과 서비스 내용이 대부분1. 식품 내용은 기존의 Category 분류 알고리즘 활용하기1. 서비스는 Infomation Architecture 를 활용한 대분류 생성하기<jupyter_code># import pandas as pd
# from muyong.nlp import food_nouns
# items_none = [_.strip() for _,v in items_maulo_category.items()  if v == '']
# with open("data/food_nouns.txt" , "r") as f:
#     nouns_token = f.read().split(",")
# menu_valid = [["_".join(food_nouns(_, nouns_token)), _]  for _ in items_none]
# pd.DataFrame(menu_valid).to_csv("items_error.csv", index=None)<jupyter_output><empty_output><jupyter_text>**2 Handwriting Fixed**알고리즘으로 해결 안되는 내용 추가작업 진행<jupyter_code>import pickle, gzip
# with open('./data/mauloItemNaver.pickle', 'rb') as f:
#     items_maulo_category = pickle.load(f)
# print("loading the date.")
# fixing the Error by Handwriting.
import pandas as pd
items_error_df = pd.read_csv('./data/maulo_items_error.csv', header=None)
items_error = sorted(set(items_error_df[0].to_list()))
result = []
for _ in items_error:
    if _.find("서비스") != -1:
        result.append(_.replace("서비스", "").strip() + " 서비스")
    else:
        result.append(_)
items_error = result
">>  ".join([_ for _ in items_error] )
# result_error_token = naver_finder(items_error)
# result = {}
# for k,v in result_error_token.items():
#     if k.find("서비스") != -1:
#         k = ("서비스 " + k.replace("서비스", "").strip())
#         result[k] = v
#     else:
#         result[k] = v
# "> ".join(result.keys())

# import pickle
# with open('./data/mauloItemNaver_error.pickle', 'wb') as f:
#     pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
# print("saving is done.")<jupyter_output><empty_output><jupyter_text>**3 ReRanging the Error Table**Handwriting Fixed Table<jupyter_code>import pickle
with open('./data/mauloItemNaver_error.pickle', 'rb') as f:
    dict_errors = pickle.load(f)
print("loading is done.")
result = {}
for _ in range(len(items_error_df)):
    result[items_error_df.iloc[_,1]] = dict_errors[items_error_df.iloc[_,0]] 
# Saving the result
with open('./data/mauloItemNaver_error_v1.pickle', 'wb') as f:
    pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
print("saving is done.")<jupyter_output>saving is done.<jupyter_text>**4 Final Result**Merging the results..<jupyter_code>import pickle, gzip
item_result = {}
with open('./data/mauloItemNaver.pickle', 'rb') as f:
    items_maulo_category = pickle.load(f)
with open('./data/mauloItemNaver_error_v1.pickle', 'rb') as f:
    items_maulo_category_error_fix_v1 = pickle.load(f)
for k,v in items_maulo_category.items():
    if v == "": 
        v = items_maulo_category_error_fix_v1[k.strip()]
    else: 
        pass
    if v == "": 
        print(k)
    item_result[k.strip()] = v

del items_maulo_category, items_maulo_category_error_fix_v1, k, v
print("loading the date.")
# Saving the result
with open('./data/category_Naver.pickle', 'wb') as f:
    pickle.dump(item_result, f, pickle.HIGHEST_PROTOCOL)
print("saving is done.")<jupyter_output>saving is done.
StarcoderdataJupyterStructuredCleanDedup