<jupyter_start><jupyter_text>Пример: кластеризация текстов<jupyter_code>import pandas as pd
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import scipy.stats as sts
import seaborn as sns
from contextlib import contextmanager

sns.set()
sns.set_style("whitegrid")

color_palette = sns.color_palette('deep') + sns.color_palette('husl', 6) + sns.color_palette('bright') + sns.color_palette('pastel')

%matplotlib inline
sns.palplot(color_palette)

def ndprint(a, precision=3):
    with np.printoptions(precision=precision, suppress=True):
        print(a)<jupyter_output><empty_output><jupyter_text>Выборка<jupyter_code>from sklearn import datasets, metrics, model_selection as mdsel, tree, ensemble, neighbors

train_all = datasets.fetch_20newsgroups(subset='train')
print(train_all.target_names)
simple_dataset = datasets.fetch_20newsgroups(
    subset='train', 
    categories=['comp.sys.mac.hardware', 'soc.religion.christian', 'rec.sport.hockey']
)
print(simple_dataset.data[0])
simple_dataset.target
print(simple_dataset.data[-1])
print(simple_dataset.data[-2])
len(simple_dataset.data)<jupyter_output><empty_output><jupyter_text>Признаки<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

vectorizer = TfidfVectorizer(max_df=500, min_df=10)
matrix = vectorizer.fit_transform(simple_dataset.data)
matrix.shape<jupyter_output><empty_output><jupyter_text>Аггломеративная кластеризация (neighbour joining)<jupyter_code>from sklearn.cluster.hierarchical import AgglomerativeClustering

model = AgglomerativeClustering(n_clusters=3, affinity='cosine', linkage='complete')
preds = model.fit_predict(matrix.toarray())
list(preds)[:20]
print(matrix[0])
vectorizer.get_feature_names()[:10]
vectorizer.get_feature_names()[877]
simple_dataset.data[0]<jupyter_output><empty_output><jupyter_text>KMeans<jupyter_code>from sklearn.cluster import KMeans

model = KMeans(n_clusters=3, random_state=1)
preds = model.fit_predict(matrix.toarray())
print(preds)
print(simple_dataset.target)
mapping = {2 : 1, 1: 2, 0: 0}
mapped_preds = [mapping[pred] for pred in preds]
print(float(sum(mapped_preds != simple_dataset.target)) / len(simple_dataset.target))
from sklearn.linear_model import LogisticRegression
clf = LogisticRegression(solver='lbfgs', multi_class='auto')
print(mdsel.cross_val_score(clf, matrix, simple_dataset.target, cv=3).mean())<jupyter_output>0.9864864479274559<jupyter_text>Более сложная выборка<jupyter_code>dataset = datasets.fetch_20newsgroups(
    subset='train', 
    categories=['comp.sys.mac.hardware', 'comp.os.ms-windows.misc', 'comp.graphics'])
matrix = vectorizer.fit_transform(dataset.data)
model = KMeans(n_clusters=3, random_state=42)
preds = model.fit_predict(matrix.toarray())
print(preds)
print(dataset.target)
mapping = {2 : 2, 1: 1, 0: 0}
mapped_preds = [mapping[pred] for pred in preds]
print(float(sum(mapped_preds != dataset.target)) / len(dataset.target))
clf = LogisticRegression(solver='lbfgs', multi_class='auto')
print(mdsel.cross_val_score(clf, matrix, dataset.target, cv=3).mean())<jupyter_output>0.9178509820019255<jupyter_text>SVD + KMeans<jupyter_code>from sklearn.decomposition import TruncatedSVD

model = KMeans(n_clusters=3, random_state=42)
svd = TruncatedSVD(n_components=1000, random_state=123)
features = svd.fit_transform(matrix)
preds = model.fit_predict(features)
print(preds)
print(dataset.target)
mapping = {0 : 2, 1: 0, 2: 1}
mapped_preds = [mapping[pred] for pred in preds]
print(float(sum(mapped_preds != dataset.target)) / len(dataset.target))
model = KMeans(n_clusters=3, random_state=42)
svd = TruncatedSVD(n_components=200, random_state=123)
features = svd.fit_transform(matrix)
preds = model.fit_predict(features)
print(preds)
print(dataset.target)
import itertools
def validate_with_mappings(preds, target, dataset):
    permutations = itertools.permutations([0, 1, 2])
    for a, b, c in permutations:
        mapping = {2 : a, 1: b, 0: c}
        mapped_preds = [mapping[pred] for pred in preds]
        print(float(sum(mapped_preds != target)) / len(target))
        
validate_with_mappings(preds, dataset.target, dataset)
model = KMeans(n_clusters=3, random_state=42)
svd = TruncatedSVD(n_components=200, random_state=321)
features = svd.fit_transform(matrix)
preds = model.fit_predict(features)
print(preds)
print(dataset.target)
validate_with_mappings(preds, dataset.target, dataset)<jupyter_output>[2 1 0 ... 2 0 2]
[2 1 1 ... 2 0 2]
0.713063320022818
0.845407872219053
0.8893325727324587
0.7005134055904164
0.5864232743867656
0.2652595550484883
StarcoderdataJupyterStructuredCleanDedup