{
"cells": [
{
"cell_type": "markdown",
"id": "4d392599",
"metadata": {},
"source": [
"\n",
"# Кластеризация и поиск аномалий\n",
"\n",
"Задача:\n",
"- решить задачу кластеризации произвольным методом\n",
"- оценить качество кластеризации\n",
"- найти и удалить наиболее аномальные объекты\n",
"- повторить кластеризацию тем же методом\n",
"- сравнить качество до и после очистки\n",
"- построить графики результатов\n",
"\n",
"Ноутбук сделан без финального содержательного вывода. \n",
"После запуска можно будет добавить короткий итог по реальным результатам.\n"
]
},
{
"cell_type": "markdown",
"id": "39161038",
"metadata": {},
"source": [
"## 1. Импорт библиотек"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "2795a57b",
"metadata": {},
"outputs": [],
"source": [
"\n",
"import warnings\n",
"warnings.filterwarnings(\"ignore\")\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"\n",
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.cluster import KMeans\n",
"from sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score\n",
"from sklearn.ensemble import IsolationForest\n",
"from sklearn.neighbors import LocalOutlierFactor\n",
"from sklearn.decomposition import PCA\n"
]
},
{
"cell_type": "markdown",
"id": "99100adc",
"metadata": {},
"source": [
"## 2. Загрузка данных и проверка столбцов"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "eaa2de1c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Размер датасета: (876, 3)\n",
"Столбцы: ['x', 'y', 'color']\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" x | \n",
" y | \n",
" color | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 355.297197 | \n",
" 148.880238 | \n",
" 1 | \n",
"
\n",
" \n",
" | 1 | \n",
" 402.111403 | \n",
" 244.566989 | \n",
" 1 | \n",
"
\n",
" \n",
" | 2 | \n",
" 383.443863 | \n",
" 224.191167 | \n",
" 1 | \n",
"
\n",
" \n",
" | 3 | \n",
" 424.774071 | \n",
" 207.348790 | \n",
" 1 | \n",
"
\n",
" \n",
" | 4 | \n",
" 398.722062 | \n",
" 214.262629 | \n",
" 1 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" x | \n",
" y | \n",
"
\n",
" \n",
" \n",
" \n",
" | count | \n",
" 876.000000 | \n",
" 876.000000 | \n",
"
\n",
" \n",
" | mean | \n",
" 312.709185 | \n",
" 177.811060 | \n",
"
\n",
" \n",
" | std | \n",
" 122.928335 | \n",
" 53.831720 | \n",
"
\n",
" \n",
" | min | \n",
" 80.929909 | \n",
" 7.121305 | \n",
"
\n",
" \n",
" | 25% | \n",
" 182.039481 | \n",
" 137.372667 | \n",
"
\n",
" \n",
" | 50% | \n",
" 373.329877 | \n",
" 176.407887 | \n",
"
\n",
" \n",
" | 75% | \n",
" 417.359332 | \n",
" 214.627433 | \n",
"
\n",
" \n",
" | max | \n",
" 510.249841 | \n",
" 406.044153 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"inertias = []\n",
"sil_scores = []\n",
"\n",
"k_range = range(2, 7)\n",
"for k in k_range:\n",
" km = KMeans(n_clusters=k, random_state=42, n_init=10)\n",
" labels = km.fit_predict(X)\n",
" inertias.append(km.inertia_)\n",
" sil_scores.append(silhouette_score(X, labels))\n",
"\n",
"best_k = list(k_range)[int(np.argmax(sil_scores))]\n",
"print(\"Лучшее число кластеров по silhouette:\", best_k)\n",
"\n",
"plt.figure(figsize=(10, 4))\n",
"\n",
"plt.subplot(1, 2, 1)\n",
"plt.plot(list(k_range), inertias, marker=\"o\")\n",
"plt.title(\"Метод локтя\")\n",
"plt.xlabel(\"k\")\n",
"plt.ylabel(\"Inertia\")\n",
"plt.grid(alpha=0.3)\n",
"\n",
"plt.subplot(1, 2, 2)\n",
"plt.plot(list(k_range), sil_scores, marker=\"o\")\n",
"plt.title(\"Silhouette score\")\n",
"plt.xlabel(\"k\")\n",
"plt.ylabel(\"Score\")\n",
"plt.grid(alpha=0.3)\n",
"\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
},
{
"cell_type": "markdown",
"id": "2eb1ed49",
"metadata": {},
"source": [
"## 5. Базовая кластеризация"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "bd174b62",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Метрика | \n",
" Значение | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" Silhouette | \n",
" 0.616345 | \n",
"
\n",
" \n",
" | 1 | \n",
" Calinski-Harabasz | \n",
" 1802.976446 | \n",
"
\n",
" \n",
" | 2 | \n",
" Davies-Bouldin | \n",
" 0.556134 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"plt.figure(figsize=(7, 6))\n",
"plt.scatter(data[\"x\"], data[\"y\"], c=labels_full, alpha=0.7)\n",
"centers_full = scaler.inverse_transform(kmeans_full.cluster_centers_)\n",
"plt.scatter(centers_full[:, 0], centers_full[:, 1], marker=\"x\", s=150, linewidths=3)\n",
"plt.title(\"Кластеры до удаления аномалий\")\n",
"plt.xlabel(\"x\")\n",
"plt.ylabel(\"y\")\n",
"plt.grid(alpha=0.3)\n",
"plt.show()\n"
]
},
{
"cell_type": "markdown",
"id": "e297ef97",
"metadata": {},
"source": [
"\n",
"## 7. Поиск аномалий\n",
"\n",
"Используем три способа:\n",
"1. `Isolation Forest`\n",
"2. `Local Outlier Factor`\n",
"3. расстояние до ближайшего центра KMeans\n",
"\n",
"Объект считается аномальным, если его отметили минимум 2 из 3 методов.\n"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "1bea67fc",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Число аномалий: 32\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" x | \n",
" y | \n",
" cluster | \n",
" flag_iso | \n",
" flag_lof | \n",
" flag_dist | \n",
" votes | \n",
" anomaly | \n",
" distance_to_center | \n",
"
\n",
" \n",
" \n",
" \n",
" | 36 | \n",
" 148.048573 | \n",
" 387.272142 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 3.957187 | \n",
"
\n",
" \n",
" | 492 | \n",
" 411.020015 | \n",
" 406.044153 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 3.698639 | \n",
"
\n",
" \n",
" | 166 | \n",
" 242.781695 | \n",
" 384.528872 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 3.558861 | \n",
"
\n",
" \n",
" | 491 | \n",
" 273.410505 | \n",
" 370.380086 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 3.224241 | \n",
"
\n",
" \n",
" | 15 | \n",
" 205.556808 | \n",
" 344.122137 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 3.029224 | \n",
"
\n",
" \n",
" | 567 | \n",
" 188.786351 | \n",
" 313.372106 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.656472 | \n",
"
\n",
" \n",
" | 348 | \n",
" 302.867610 | \n",
" 7.121305 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.584363 | \n",
"
\n",
" \n",
" | 407 | \n",
" 510.249841 | \n",
" 81.167665 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.486561 | \n",
"
\n",
" \n",
" | 338 | \n",
" 241.912663 | \n",
" 318.832690 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.473812 | \n",
"
\n",
" \n",
" | 682 | \n",
" 332.991943 | \n",
" 34.117597 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.274358 | \n",
"
\n",
" \n",
" | 677 | \n",
" 437.288253 | \n",
" 92.794569 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.138226 | \n",
"
\n",
" \n",
" | 363 | \n",
" 475.145599 | \n",
" 313.340803 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 2.051985 | \n",
"
\n",
" \n",
" | 159 | \n",
" 131.512996 | \n",
" 232.495301 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 1.870437 | \n",
"
\n",
" \n",
" | 118 | \n",
" 414.128940 | \n",
" 107.250885 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 1.856117 | \n",
"
\n",
" \n",
" | 560 | \n",
" 415.484492 | \n",
" 301.876330 | \n",
" 0 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 1.763698 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"plt.figure(figsize=(7, 6))\n",
"plt.scatter(data[\"x\"], data[\"y\"], c=labels_full, alpha=0.5, label=\"Объекты\")\n",
"idx = anomaly_df[anomaly_df[\"anomaly\"] == 1].index\n",
"if len(idx) > 0:\n",
" plt.scatter(data.loc[idx, \"x\"], data.loc[idx, \"y\"], marker=\"x\", s=90, label=\"Аномалии\")\n",
"plt.title(\"Аномалии на исходных данных\")\n",
"plt.xlabel(\"x\")\n",
"plt.ylabel(\"y\")\n",
"plt.legend()\n",
"plt.grid(alpha=0.3)\n",
"plt.show()\n"
]
},
{
"cell_type": "markdown",
"id": "d8834f45",
"metadata": {},
"source": [
"## 9. Удаление аномалий"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "d52789fe",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Размер до очистки: 876\n",
"Размер после очистки: 844\n"
]
}
],
"source": [
"\n",
"clean_data = anomaly_df[anomaly_df[\"anomaly\"] == 0][feature_cols].copy()\n",
"\n",
"scaler_clean = StandardScaler()\n",
"X_clean = scaler_clean.fit_transform(clean_data)\n",
"\n",
"print(\"Размер до очистки:\", len(data))\n",
"print(\"Размер после очистки:\", len(clean_data))\n"
]
},
{
"cell_type": "markdown",
"id": "72aade2a",
"metadata": {},
"source": [
"## 10. Повторная кластеризация"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "fd4d546a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Метрика | \n",
" Значение | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" Silhouette | \n",
" 0.626347 | \n",
"
\n",
" \n",
" | 1 | \n",
" Calinski-Harabasz | \n",
" 2064.894424 | \n",
"
\n",
" \n",
" | 2 | \n",
" Davies-Bouldin | \n",
" 0.540785 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"plt.figure(figsize=(7, 6))\n",
"plt.scatter(clean_data[\"x\"], clean_data[\"y\"], c=labels_clean, alpha=0.7)\n",
"centers_clean = scaler_clean.inverse_transform(kmeans_clean.cluster_centers_)\n",
"plt.scatter(centers_clean[:, 0], centers_clean[:, 1], marker=\"x\", s=150, linewidths=3)\n",
"plt.title(\"Кластеры после удаления аномалий\")\n",
"plt.xlabel(\"x\")\n",
"plt.ylabel(\"y\")\n",
"plt.grid(alpha=0.3)\n",
"plt.show()\n"
]
},
{
"cell_type": "markdown",
"id": "3c001388",
"metadata": {},
"source": [
"## 12. Сравнение качества кластеризации"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "6df72aca",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Метрика | \n",
" До очистки | \n",
" После очистки | \n",
" Изменение | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" Silhouette | \n",
" 0.616345 | \n",
" 0.626347 | \n",
" 0.010002 | \n",
"
\n",
" \n",
" | 1 | \n",
" Calinski-Harabasz | \n",
" 1802.976446 | \n",
" 2064.894424 | \n",
" 261.917978 | \n",
"
\n",
" \n",
" | 2 | \n",
" Davies-Bouldin | \n",
" 0.556134 | \n",
" 0.540785 | \n",
" -0.015348 | \n",
"
\n",
" \n",
"
\n",
"