<jupyter_start><jupyter_text>Import Modules<jupyter_code>import docx
from docx import Document

import pandas as pd
import numpy as np
from collections import defaultdict

from scipy.spatial import distance
from scipy.cluster import hierarchy

from sklearn import decomposition
from sklearn import preprocessing
from sklearn.metrics import silhouette_score

import seaborn as sns
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Read Data in Docx Format<jupyter_code>document = Document('datasets/data.docx')
table = document.tables[1]<jupyter_output><empty_output><jupyter_text>Convert Docx Table to Dataframe<jupyter_code>data = np.array([
    [cell.text for cell in row.cells]
    for row in table.rows[4:]
])
df = pd.DataFrame(
    data=data, 
    columns=['region', 'turnover', 'first', 'second', 'third']
)
df.set_index('region', inplace=True)

df = df.applymap(
    lambda x: x.replace(',', '.')
).astype(float)
df<jupyter_output><empty_output><jupyter_text>EDA  Retail Turnover in different Ukrainian Regions<jupyter_code>plt.figure(figsize=(10, 10))
sns.barplot(
    x=df.turnover,
    y=df.index,
    order=df.sort_values('turnover', ascending=False).index,
    palette='viridis'
)
plt.show()<jupyter_output><empty_output><jupyter_text>How does Turnover depend on Rate of Change?<jupyter_code>sns.pairplot(df, corner=True, kind='kde')<jupyter_output><empty_output><jupyter_text>Hierarchical Clustering  Visualize Data<jupyter_code>scaler = preprocessing.StandardScaler()
norm_df = scaler.fit_transform(df)

pca = decomposition.PCA(n_components=2, random_state=42)
x, y = pca.fit_transform(norm_df).T

plt.figure(figsize=(6, 6));
plt.scatter(x=x, y=y, color='green')<jupyter_output><empty_output><jupyter_text>Compute Distances<jupyter_code>metrics = ['euclidean', 'seuclidean', 'cityblock', 'mahalanobis', 'minkowski', 'chebyshev']
n_metrics = len(metrics)
pw_distances = {
    metric: distance.pdist(
        X=norm_df,
        metric=metric, 
        **({'p': 4} if metric == 'minkowski' else dict())
    )
    for metric in metrics
}<jupyter_output><empty_output><jupyter_text>Visualize Distances<jupyter_code>def plot_distance_matrix(metric: str) -> None:
    plt.figure(figsize=(16, 16))
    
    matrix = distance.squareform(pw_distances[metric])
    sns.heatmap(matrix, annot=True)
plot_distance_matrix('cityblock')<jupyter_output><empty_output><jupyter_text>Clusterize Data and Compute Cophenet Correlation Coefficients<jupyter_code>methods = ['single', 'complete', 'average', 'centroid', 'median']
n_methods = len(methods)
cophenets = np.zeros((n_metrics, n_methods))
clustereds = defaultdict(dict)

for didx, (metric, pw_distance) in enumerate(pw_distances.items()):
    for lidx, method in enumerate(methods):
        
        clustered = hierarchy.linkage(pw_distance, method)
        cophenet = hierarchy.cophenet(clustered, pw_distance)[0]
        
        clustereds[metric][method] = clustered
        cophenets[didx][lidx] = cophenet

cophenet_df = pd.DataFrame(cophenets, metrics, methods)
cophenet_df<jupyter_output><empty_output><jupyter_text>Find Best and Worst (Methods + Metrics)<jupyter_code>pd.DataFrame([cophenet_df.idxmin(), cophenet_df.min()])
pd.DataFrame([cophenet_df.idxmax(), cophenet_df.max()])<jupyter_output><empty_output><jupyter_text>Dendrogram For Clustering<jupyter_code>best_metric = 'euclidean'
best_method = 'average'
best_clustered = clustereds[best_metric][best_method]
def pretty_dendrogram(metric: str, method: str, **kwargs) -> dict[str, np.ndarray]:
    plt.figure(figsize=(12, 15))

    max_depth = kwargs.pop('max_depth', None)
    if max_depth:
        kwargs['color_threshold'] = max_depth
    anot_height = kwargs.pop('annot_height', 0.5)
    
    dn = hierarchy.dendrogram(
        Z=clustereds[metric][method],
        labels=df.index, **kwargs
    )

    icoords = dn['icoord']
    dcoords = dn['dcoord']
    colors = dn['color_list']

    for icoord, dcoord, color in zip(icoords, dcoords, colors):
        xx = np.sum(icoord[1:3]) / 2
        yy = dcoord[1]
        if yy < anot_height: continue
        plt.plot(xx, yy, marker='o', color=color)
        plt.annotate(
            text='{:.3f}'.format(yy),
            xy=(xx, yy),
            xytext=(0, -5),
            textcoords='offset points',
            va='top',
            ha='center'
        )
    if max_depth:
        plt.axhline(y=max_depth, color='black')
        
    plt.title(f'''
        Metric: {metric.title()} 
        Method: {method.title()}
    ''')
    return dn
ddata = pretty_dendrogram(
    metric=best_metric,
    method=best_method,
    max_depth=2.5, 
    annot_height=0.3
)<jupyter_output><empty_output><jupyter_text>Elbow Method<jupyter_code>def elbow_method(metric: str, method: str, min_clusters: int = 1, max_clusters: int = 10) -> int:
    clustered = clustereds[metric][method]
    
    n_clusters = np.arange(min_clusters, max_clusters)
    inertias = np.zeros_like(n_clusters)
    
    for idx, num in enumerate(n_clusters):
        targets = hierarchy.fcluster(
            clustered, num, 'maxclust'
        )
        result = pd.DataFrame(norm_df)
        result['target'] = targets
        
        centroids = result.groupby('target').agg(np.mean).values
        inertia = np.square(
            norm_df - centroids[targets - 1]).sum()
        
        inertias[idx] = inertia
    
    def plot_elbow() -> None:
        plt.figure(figsize=(8, 8))
        
        plt.xlabel('Num Clusters')
        plt.ylabel('Inertia')
        
        plt.plot(n_clusters, inertias, '-ro')
    
    plot_elbow()
    return inertias
elbow_method(best_metric, best_method)<jupyter_output><empty_output><jupyter_text>Split on Clusters<jupyter_code>n_clusters = 6
targets = hierarchy.fcluster(
    Z=best_clustered,
    t=n_clusters, 
    criterion='maxclust'
)<jupyter_output><empty_output><jupyter_text>Compute Statistics<jupyter_code>target_df = pd.DataFrame(norm_df, columns=df.columns)
target_df['target'] = targets<jupyter_output><empty_output><jupyter_text>Centroids<jupyter_code>centroids = target_df.groupby('target').agg(np.mean).values
centroids<jupyter_output><empty_output><jupyter_text>Distances Between Centroids<jupyter_code>distance.pdist(centroids)<jupyter_output><empty_output><jupyter_text>Silhouette Score<jupyter_code>silhouette_score(
    X=distance.squareform(pw_distances[best_metric]),
    labels=targets
)<jupyter_output><empty_output><jupyter_text>Visualize Clustered Data<jupyter_code>target2d = pd.DataFrame({'x': x, 'y': y, 'target': targets})

plt.figure(figsize=(8, 6))
plt.scatter(x, y, c=targets, s=50)

for idx, label in enumerate(df.index):
    plt.annotate(
        text=label, 
        xy=(x[idx] + 0.1, y[idx])
    )
plt.scatter(
    x='x', y='y', s=200, 
    color='r', 
    marker='x', 
    data=target2d.groupby('target').agg('mean')
)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup