<jupyter_start><jupyter_text>数据清洗1. 全角 --> 半角； 英文统一小写2. 去除 html tag, 章节名，括号内容，url 链接 （及变体）3. 重复字、符合并<jupyter_code>book = data_path/"唐家三少"/"斗罗大陆.txt"
text = book.read_text()[:1000]
text
from prepare_data import clean_text, split_sentence
from hanlp.utils.lang.zh.char_table import CharTable
new_text = clean_text(text)
new_text<jupyter_output><empty_output><jupyter_text>数据分析 1. 句子段落长度分析2. 标准化存储分词数据集3. 词频分析<jupyter_code>import json
def load_book_res():
    book_results = {}
    for author in book_list:
        author_dir = data_path/author
        for book in book_list[author]:
            book_res_file = author_dir/"{}.json".format(book)
            with book_res_file.open(mode="r") as f:
                book_res = json.load(f)
            book_results[book] = book_res
    return book_results
book_results = load_book_res()
import random
random.choice(book_results["酒神"]["tokens"])<jupyter_output><empty_output><jupyter_text>按照分词计算句子长度<jupyter_code>book_sentences = []
for book, res in book_results.items():
    book_sentences.extend(res["tokens"])

sentences_len = [len(x) for x in book_sentences]
sorted_sentences = sorted(zip(book_sentences, sentences_len), key=lambda x: x[1], reverse=True)
print("Total sentences number: {}".format(len(sorted_sentences)))
import seaborn as sns
import numpy as np
sns.distplot(np.clip(sentences_len, 0, 150), kde=True, )
sns.distplot(sentences_len, rug=True, )
random.choice(sorted_sentences[:100])<jupyter_output><empty_output><jupyter_text>按照字符串长度计算句子长度<jupyter_code>book_sentences_char = []
for book, res in book_results.items():
    book_sentences_char.extend(res["sentences"])
    
sentences_char_len = [len(x) for x in book_sentences_char]
sorted_sentences_char = sorted(zip(book_sentences_char, sentences_char_len), key=lambda x: x[1], reverse=True)
sns.distplot(sentences_char_len, rug=True)
sns.distplot(np.clip(sentences_char_len, 0, 300), kde=True)
random.choice(sorted_sentences_char[:1000])<jupyter_output><empty_output><jupyter_text>统计 comma seperator 下的句子长度<jupyter_code>book_sentences_char_comma = []
for book, res in book_results.items():
    sentences = res["sentences"]
    sentences_comma = []
    for sentence in sentences:
        sentences_comma.extend(split_sentence(sentence, comma=True))
    book_sentences_char_comma.extend(sentences_comma)
    
sentences_char_comma_len = [len(x) for x in book_sentences_char_comma]
sorted_sentences_char_comma = sorted(zip(book_sentences_char_comma, sentences_char_comma_len), key=lambda x: x[1], reverse=True)
print("Total number of comma separated sentences = {}".format(len(sentences_char_comma_len)))
sns.distplot(sentences_char_comma_len, kde=True, rug=True)
sns.distplot(np.clip(sentences_char_comma_len, 0, 110))
random.choice(sorted_sentences_char_comma[:1000])<jupyter_output><empty_output><jupyter_text>统计对比分词与不分词情况下的句子序列长度分布<jupyter_code>import numpy as np
def show_list_stat(lst):
    quantile1 = np.quantile(lst, 0.01)
    quantile10 = np.quantile(lst, 0.1)
    quantile20 = np.quantile(lst, 0.2)
    median = np.median(lst)
    quantile90 = np.quantile(lst, 0.9)
    quantile99 = np.quantile(lst, 0.99)
    quantile999 = np.quantile(lst,0.999)
    quantile9999 = np.quantile(lst,0.9999)
    print("1% Quantile =\t\t {}\n10% Quantile =\t\t {}\n20% Quantile =\t\t {}\n"
          "Median =\t\t {}\n90% Quantile =\t\t {}\n99% Quantile =\t\t {}\n99.9% Quantile"
          " =\t {}\n99.99% Quantile =\t {}".format(
        quantile1, quantile10, quantile20, median, quantile90, quantile99, quantile999, quantile9999))
# word level
show_list_stat(sentences_len)
# char level
show_list_stat(sentences_char_len)
# char level with comma separator
show_list_stat(sentences_char_comma_len)<jupyter_output>1% Quantile =		 2.0
10% Quantile =		 5.0
20% Quantile =		 6.0
Median =		 10.0
90% Quantile =		 18.0
99% Quantile =		 28.0
99.9% Quantile =	 47.0
99.99% Quantile =	 105.0<jupyter_text>Try to remove \. inside sentences, but cannot apply this to text cleaning. Because there're legal words like U.S., V.S., etc.<jupyter_code># remove the \. inside sentences
# cannot do this trivially, there's legal words like U.S., V.S., etc.
test_text = "我喜欢你.是我度假的集以。安德森发.asdfasdf"
import re
re.sub(r"([^\.])(\.)([^\.])", r"\1\3", test_text)
# test split sentence with ,
long_sentence = "轻轻杞玩着手\n中卷轴,萧炎!微微皱。着眉,这种具有着改变人体质的奇丹.炼制。起来！困难度极大,失败率也极高,而且最可怕的是,这种阶别的丹药,在出炉之时,有着几率会引起天地能量波动,最后引发出雷劫,这种雷劫,炼药界中又称之为丹劫,威力极大,一个不慎,便是丹毁人亡的下场,因此,即便是一些有能力炼制七品丹药的炼药师,也会尽量避免少炼制这种会引发丹劫的丹药,而这也能猜测出,为什么这么多年,蛇人族从来没有弄出一枚天魂融血丹了。"
list(split_sentence(long_sentence, comma=True))<jupyter_output><empty_output><jupyter_text>标准化储存分词后数据集<jupyter_code>book_cws_text = {}
for author in book_list:
    author_dir = data_path / author
    for book in book_list[author]:
        book_res_file = author_dir/"{}.json".format(book)
        with book_res_file.open(mode="r") as f:
            book_res = json.load(f)
        tokens = book_res["tokens"]
        text = "\n".join([" ".join(sentence) for sentence in tokens])
        book_cws_file = author_dir/"{}.cws.txt".format(book)
        book_cws_file.write_text(text, encoding="utf-8")
        book_cws_text[book] = text

"\n".join([" ".join(sentence) for sentence in tokens[:10]])
long_sentence = "轻轻杞玩着手中卷轴,萧炎微微皱着眉,这种具有着改变人体质的奇丹,炼制起来困难度极大,失败率也极高,而且最可怕的是,这种阶别的丹药,在出炉之时,有着几率会引起天地能量波动,最后引发出雷劫,这种雷劫,炼药界中又称之为丹劫,威力极大,一个不慎,便是丹毁人亡的下场,因此,即便是一些有能力炼制七品丹药的炼药师,也会尽量避免少炼制这种会引发丹劫的丹药,而这也能猜测出,为什么这么多年,蛇人族从来没有弄出一枚天魂融血丹了。"
long_sentence.split(",")<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup