{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Лабораторная работа №5\n",
    "\n",
    "*Вариант задания:* "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['summary', 'city', 'state', 'date_time', 'shape', 'duration', 'stats',\n",
      "       'report_link', 'text', 'posted', 'city_latitude', 'city_longitude'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.cluster.hierarchy import dendrogram, linkage, fcluster\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import silhouette_score\n",
    "\n",
    "df = pd.read_csv(\"../../datasets/nuforc_reports.csv\")\n",
    "df = df.iloc[:1000].dropna()\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>date_time</th>\n",
       "      <th>shape</th>\n",
       "      <th>duration</th>\n",
       "      <th>stats</th>\n",
       "      <th>report_link</th>\n",
       "      <th>text</th>\n",
       "      <th>posted</th>\n",
       "      <th>city_latitude</th>\n",
       "      <th>city_longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Viewed some red lights in the sky appearing to...</td>\n",
       "      <td>Visalia</td>\n",
       "      <td>CA</td>\n",
       "      <td>2021-12-15T21:45:00</td>\n",
       "      <td>light</td>\n",
       "      <td>2 minutes</td>\n",
       "      <td>Occurred : 12/15/2021 21:45  (Entered as : 12/...</td>\n",
       "      <td>http://www.nuforc.org/webreports/165/S165881.html</td>\n",
       "      <td>Viewed some red lights in the sky appearing to...</td>\n",
       "      <td>2021-12-19T00:00:00</td>\n",
       "      <td>36.356650</td>\n",
       "      <td>-119.347937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Look like 1 or 3 crafts from North traveling s...</td>\n",
       "      <td>Cincinnati</td>\n",
       "      <td>OH</td>\n",
       "      <td>2021-12-16T09:45:00</td>\n",
       "      <td>triangle</td>\n",
       "      <td>14 seconds</td>\n",
       "      <td>Occurred : 12/16/2021 09:45  (Entered as : 12/...</td>\n",
       "      <td>http://www.nuforc.org/webreports/165/S165888.html</td>\n",
       "      <td>Look like 1 or 3 crafts from North traveling s...</td>\n",
       "      <td>2021-12-19T00:00:00</td>\n",
       "      <td>39.174503</td>\n",
       "      <td>-84.481363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>One red light moving switly west to east, beco...</td>\n",
       "      <td>Knoxville</td>\n",
       "      <td>TN</td>\n",
       "      <td>2021-12-10T19:30:00</td>\n",
       "      <td>triangle</td>\n",
       "      <td>20-30 seconds</td>\n",
       "      <td>Occurred : 12/10/2021 19:30  (Entered as : 12/...</td>\n",
       "      <td>http://www.nuforc.org/webreports/165/S165825.html</td>\n",
       "      <td>One red light moving switly west to east, beco...</td>\n",
       "      <td>2021-12-19T00:00:00</td>\n",
       "      <td>35.961561</td>\n",
       "      <td>-83.980115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>I'm familiar with all the fakery and UFO sight...</td>\n",
       "      <td>Fullerton</td>\n",
       "      <td>CA</td>\n",
       "      <td>2020-07-07T23:00:00</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2 minutes</td>\n",
       "      <td>Occurred : 7/7/2020 23:00  (Entered as : 07/07...</td>\n",
       "      <td>http://www.nuforc.org/webreports/157/S157444.html</td>\n",
       "      <td>I'm familiar with all the fakery and UFO sight...</td>\n",
       "      <td>2020-07-09T00:00:00</td>\n",
       "      <td>33.877422</td>\n",
       "      <td>-117.924978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I was driving up lakes mead towards the lake a...</td>\n",
       "      <td>Las Vegas</td>\n",
       "      <td>NV</td>\n",
       "      <td>2020-04-23T03:00:00</td>\n",
       "      <td>oval</td>\n",
       "      <td>10 minutes</td>\n",
       "      <td>Occurred : 4/23/2020 03:00  (Entered as : 4/23...</td>\n",
       "      <td>http://www.nuforc.org/webreports/155/S155608.html</td>\n",
       "      <td>I was driving up lakes mead towards the lake a...</td>\n",
       "      <td>2020-05-01T00:00:00</td>\n",
       "      <td>36.141246</td>\n",
       "      <td>-115.186592</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             summary        city state  \\\n",
       "0  Viewed some red lights in the sky appearing to...     Visalia    CA   \n",
       "1  Look like 1 or 3 crafts from North traveling s...  Cincinnati    OH   \n",
       "3  One red light moving switly west to east, beco...   Knoxville    TN   \n",
       "5  I'm familiar with all the fakery and UFO sight...   Fullerton    CA   \n",
       "6  I was driving up lakes mead towards the lake a...   Las Vegas    NV   \n",
       "\n",
       "             date_time     shape       duration  \\\n",
       "0  2021-12-15T21:45:00     light      2 minutes   \n",
       "1  2021-12-16T09:45:00  triangle     14 seconds   \n",
       "3  2021-12-10T19:30:00  triangle  20-30 seconds   \n",
       "5  2020-07-07T23:00:00   unknown      2 minutes   \n",
       "6  2020-04-23T03:00:00      oval     10 minutes   \n",
       "\n",
       "                                               stats  \\\n",
       "0  Occurred : 12/15/2021 21:45  (Entered as : 12/...   \n",
       "1  Occurred : 12/16/2021 09:45  (Entered as : 12/...   \n",
       "3  Occurred : 12/10/2021 19:30  (Entered as : 12/...   \n",
       "5  Occurred : 7/7/2020 23:00  (Entered as : 07/07...   \n",
       "6  Occurred : 4/23/2020 03:00  (Entered as : 4/23...   \n",
       "\n",
       "                                         report_link  \\\n",
       "0  http://www.nuforc.org/webreports/165/S165881.html   \n",
       "1  http://www.nuforc.org/webreports/165/S165888.html   \n",
       "3  http://www.nuforc.org/webreports/165/S165825.html   \n",
       "5  http://www.nuforc.org/webreports/157/S157444.html   \n",
       "6  http://www.nuforc.org/webreports/155/S155608.html   \n",
       "\n",
       "                                                text               posted  \\\n",
       "0  Viewed some red lights in the sky appearing to...  2021-12-19T00:00:00   \n",
       "1  Look like 1 or 3 crafts from North traveling s...  2021-12-19T00:00:00   \n",
       "3  One red light moving switly west to east, beco...  2021-12-19T00:00:00   \n",
       "5  I'm familiar with all the fakery and UFO sight...  2020-07-09T00:00:00   \n",
       "6  I was driving up lakes mead towards the lake a...  2020-05-01T00:00:00   \n",
       "\n",
       "   city_latitude  city_longitude  \n",
       "0      36.356650     -119.347937  \n",
       "1      39.174503      -84.481363  \n",
       "3      35.961561      -83.980115  \n",
       "5      33.877422     -117.924978  \n",
       "6      36.141246     -115.186592  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city_latitude</th>\n",
       "      <th>city_longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>712.000000</td>\n",
       "      <td>712.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>39.350240</td>\n",
       "      <td>-97.057660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.558375</td>\n",
       "      <td>17.807918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>25.774143</td>\n",
       "      <td>-149.336500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>34.950725</td>\n",
       "      <td>-116.385628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>39.597100</td>\n",
       "      <td>-93.326900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>43.262550</td>\n",
       "      <td>-82.476700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>61.214900</td>\n",
       "      <td>-61.260300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       city_latitude  city_longitude\n",
       "count     712.000000      712.000000\n",
       "mean       39.350240      -97.057660\n",
       "std         5.558375       17.807918\n",
       "min        25.774143     -149.336500\n",
       "25%        34.950725     -116.385628\n",
       "50%        39.597100      -93.326900\n",
       "75%        43.262550      -82.476700\n",
       "max        61.214900      -61.260300"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "summary           0\n",
      "city              0\n",
      "state             0\n",
      "date_time         0\n",
      "shape             0\n",
      "duration          0\n",
      "stats             0\n",
      "report_link       0\n",
      "text              0\n",
      "posted            0\n",
      "city_latitude     0\n",
      "city_longitude    0\n",
      "dtype: int64\n",
      "summary           False\n",
      "city              False\n",
      "state             False\n",
      "date_time         False\n",
      "shape             False\n",
      "duration          False\n",
      "stats             False\n",
      "report_link       False\n",
      "text              False\n",
      "posted            False\n",
      "city_latitude     False\n",
      "city_longitude    False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "# Процент пропущенных значений признаков\n",
    "for i in df.columns:\n",
    "    null_rate = df[i].isnull().sum() / len(df) * 100\n",
    "    if null_rate > 0:\n",
    "        print(f'{i} Процент пустых значений: %{null_rate:.2f}')\n",
    "\n",
    "print(df.isnull().sum())\n",
    "\n",
    "print(df.isnull().any())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "summary            object\n",
       "city               object\n",
       "state              object\n",
       "date_time          object\n",
       "shape              object\n",
       "duration           object\n",
       "stats              object\n",
       "report_link        object\n",
       "text               object\n",
       "posted             object\n",
       "city_latitude     float64\n",
       "city_longitude    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Проверка типов столбцов\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Атрибуты  \n",
    "\n",
    "### **Атрибуты и их описание**\n",
    "\n",
    " **`city` (object)**  \n",
    "   - **Описание**: Город, где произошло наблюдение.  \n",
    "   - **Применение**: Географическая группировка или создание карт по популярности наблюдений в различных городах.\n",
    "\n",
    " **`state` (object)**  \n",
    "   - **Описание**: Штат, где произошло наблюдение.  \n",
    "   - **Применение**: Анализ распределения по штатам, создание региональных кластеров наблюдений.\n",
    "\n",
    " **`city_latitude` (float64)**  \n",
    "    - **Описание**: Географическая широта города наблюдения.  \n",
    "    - **Применение**: Географический анализ и кластеризация наблюдений.\n",
    "\n",
    " **`city_longitude` (float64)**  \n",
    "    - **Описание**: Географическая долгота города наблюдения.  \n",
    "    - **Применение**: Географический анализ и визуализация наблюдений на карте.\n",
    "\n",
    "\n",
    "### **Бизнес-цель**: \n",
    "**Выявление географических кластеров активности наблюдений НЛО для исследования и стратегического планирования.\n",
    "#### **Описание цели**:\n",
    "Разделить географические области (широты) на кластеры, чтобы определить зоны с разной интенсивностью наблюдений. \n",
    "\n",
    "\n",
    "#### **Потенциальное применение**:\n",
    "- Научные организации могут использовать результаты для анализа феноменов НЛО.\n",
    "- Туристические компании могут разработать маршруты в популярные зоны наблюдений.\n",
    "- СМИ и исследовательские проекты могут строить прогнозы о возможных местах будущих наблюдений.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Очистка данных"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Цель: Упростить набор данных, удалив несущественные столбцы, чтобы сосредоточиться на ключевых атрибутах, которые будут использоваться для кластеризации и анализа.\n",
    "\n",
    "Столбцы (такието) несущественны для анализа, они не содержат ценной информации для решения задачи."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         city state            date_time     shape               posted  \\\n",
      "0     Visalia    CA  2021-12-15T21:45:00     light  2021-12-19T00:00:00   \n",
      "1  Cincinnati    OH  2021-12-16T09:45:00  triangle  2021-12-19T00:00:00   \n",
      "3   Knoxville    TN  2021-12-10T19:30:00  triangle  2021-12-19T00:00:00   \n",
      "5   Fullerton    CA  2020-07-07T23:00:00   unknown  2020-07-09T00:00:00   \n",
      "6   Las Vegas    NV  2020-04-23T03:00:00      oval  2020-05-01T00:00:00   \n",
      "\n",
      "   city_latitude  city_longitude  \n",
      "0      36.356650     -119.347937  \n",
      "1      39.174503      -84.481363  \n",
      "3      35.961561      -83.980115  \n",
      "5      33.877422     -117.924978  \n",
      "6      36.141246     -115.186592  \n"
     ]
    }
   ],
   "source": [
    "# Удаление несущественных столбцов\n",
    "columns_to_drop = [ \"summary\", \"stats\", \"report_link\", \"duration\", \"text\"]  # Столбцы, которые можно удалить\n",
    "#\"date_time\", \"posted\", \"city\", \"state\",\n",
    "df_cleaned = df.drop(columns=columns_to_drop)\n",
    "\n",
    "print(df_cleaned.head())  # Вывод очищенного DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Визуализация парных взаимосвязей\n",
    "Визуализировать ключевые атрибуты миллиардеров для выявления закономерностей и связей между ними."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 976456 characters]",
      "text/plain": [
       "<Figure size 1600x4500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Настройка стиля графиков\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "# Создание фигуры\n",
    "plt.figure(figsize=(16, 45))\n",
    "\n",
    "# График 1: Категория vs Цена\n",
    "plt.subplot(4, 1, 1)\n",
    "sns.scatterplot(x=df_cleaned['state'], y=df_cleaned['city_latitude'], alpha=0.6, color='blue')\n",
    "plt.title('state vs city_latitude')\n",
    "plt.xlabel('state')\n",
    "plt.ylabel('city_latitude')\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "# График 2: Подкатегория vs Цена\n",
    "plt.subplot(4, 1, 2)\n",
    "sns.boxplot(x=df_cleaned['city'], y=df_cleaned['city_latitude'], color='green')\n",
    "plt.title('city vs city_latitude')\n",
    "plt.xlabel('city')\n",
    "plt.ylabel('city_latitude')\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "# График 3: Категория vs Подкатегория\n",
    "plt.subplot(4, 1, 3)\n",
    "sns.countplot(x=df_cleaned['city'], hue=df_cleaned['state'], palette='Set3')\n",
    "plt.title('state vs city')\n",
    "plt.xlabel('state')\n",
    "plt.ylabel('Count')\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "# Упорядочиваем графики\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Стандартизация данных для кластеризации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   city_latitude  state_AB  state_AK  state_AL  state_AR  state_AZ  state_BC  \\\n",
      "0      -0.538951       0.0       0.0       0.0       0.0       0.0       0.0   \n",
      "1      -0.031639       0.0       0.0       0.0       0.0       0.0       0.0   \n",
      "2      -0.610081       0.0       0.0       0.0       0.0       0.0       0.0   \n",
      "3      -0.985300       0.0       0.0       0.0       0.0       0.0       0.0   \n",
      "4      -0.577732       0.0       0.0       0.0       0.0       0.0       0.0   \n",
      "\n",
      "   state_CA  state_CO  state_CT  ...  city_Winchester  city_Winnsboro  \\\n",
      "0       1.0       0.0       0.0  ...              0.0             0.0   \n",
      "1       0.0       0.0       0.0  ...              0.0             0.0   \n",
      "2       0.0       0.0       0.0  ...              0.0             0.0   \n",
      "3       1.0       0.0       0.0  ...              0.0             0.0   \n",
      "4       0.0       0.0       0.0  ...              0.0             0.0   \n",
      "\n",
      "   city_Winston  city_Woodburn  city_Woodland  city_Woodland Park  \\\n",
      "0           0.0            0.0            0.0                 0.0   \n",
      "1           0.0            0.0            0.0                 0.0   \n",
      "2           0.0            0.0            0.0                 0.0   \n",
      "3           0.0            0.0            0.0                 0.0   \n",
      "4           0.0            0.0            0.0                 0.0   \n",
      "\n",
      "   city_Woonsocket  city_Yarmouth  city_Yelm  city_Yuma  \n",
      "0              0.0            0.0        0.0        0.0  \n",
      "1              0.0            0.0        0.0        0.0  \n",
      "2              0.0            0.0        0.0        0.0  \n",
      "3              0.0            0.0        0.0        0.0  \n",
      "4              0.0            0.0        0.0        0.0  \n",
      "\n",
      "[5 rows x 636 columns]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "import pandas as pd\n",
    "\n",
    "# Выделяем числовые и категориальные признаки\n",
    "numerical_cols = ['city_latitude']\n",
    "categorical_cols = ['state', 'city']\n",
    "\n",
    "# Масштабирование числовых признаков\n",
    "scaler = StandardScaler()\n",
    "df_numerical_scaled = scaler.fit_transform(df_cleaned[numerical_cols])\n",
    "\n",
    "# Кодирование категориальных признаков с помощью OneHotEncoder\n",
    "encoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False)  # sparse=False для удобства\n",
    "encoded_data = encoder.fit_transform(df_cleaned[categorical_cols])\n",
    "\n",
    "# Создаем новые столбцы для закодированных категориальных признаков\n",
    "encoded_df = pd.DataFrame(encoded_data, columns=encoder.get_feature_names_out(categorical_cols))\n",
    "\n",
    "# Объединяем числовые и закодированные категориальные данные\n",
    "df_encoded = pd.concat([pd.DataFrame(df_numerical_scaled, columns=numerical_cols), encoded_df], axis=1)\n",
    "\n",
    "# Выводим результат\n",
    "print(df_encoded.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 120772 characters]",
      "text/plain": [
       "<Figure size 1600x1200 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Применение PCA ТОЛЬКО к числовым данным\n",
    "pca = PCA(n_components=1)\n",
    "kc_pca = pca.fit_transform(df_numerical_scaled)\n",
    "\n",
    "# Визуализация\n",
    "plt.figure(figsize=(16, 12))\n",
    "plt.scatter(range(len(kc_pca)), kc_pca, alpha=0.6)\n",
    "plt.title(\"PCA Visualization of city_latitude Feature\")\n",
    "plt.xlabel(\"Sample Index\")\n",
    "plt.ylabel(\"Principal Component 1\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Агломеративная (иерархическая) кластеризация"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 99248 characters]",
      "text/plain": [
       "<Figure size 1600x1200 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "# Построение дендрограммы (только для числовых данных)\n",
    "linkage_matrix = linkage(df_numerical_scaled, method='ward')\n",
    "\n",
    "plt.figure(figsize=(16, 12))\n",
    "dendrogram(linkage_matrix)\n",
    "plt.title('Дендрограмма агломеративной кластеризации (числовой признак \"city_latitude\")')\n",
    "plt.xlabel('Индекс образца')\n",
    "plt.ylabel('Расстояние')\n",
    "plt.show()\n",
    "\n",
    "# Получение результатов кластеризации (только для числовых данных)\n",
    "result = fcluster(linkage_matrix, t=100, criterion='distance') \n",
    "print(result)  # Вывод результатов кластеризации (номера кластеров для каждого образца)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 91592 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Выбираем подмножество данных для кластеризации\n",
    "features = df_encoded[['city_latitude']]\n",
    "\n",
    "# Масштабирование числовых признаков\n",
    "scaled_features = scaler.fit_transform(features)\n",
    "\n",
    "# Построение дендрограммы\n",
    "linkage_matrix = linkage(scaled_features, method='ward') \n",
    "\n",
    "plt.figure(figsize=(12, 8))\n",
    "dendrogram(linkage_matrix, labels=df.index, leaf_rotation=90, leaf_font_size=10)\n",
    "plt.title('Иерархическая кластеризация (дендрограмма) по цене')\n",
    "plt.xlabel('Индекс товара')\n",
    "plt.ylabel('Евклидово расстояние')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Визуализация распределения кластеров**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Danil\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\joblib\\externals\\loky\\backend\\context.py:136: UserWarning: Could not find the number of physical cores for the following reason:\n",
      "found 0 physical cores < 1\n",
      "Returning the number of logical cores instead. You can silence this warning by setting LOKY_MAX_CPU_COUNT to the number of cores you want to use.\n",
      "  warnings.warn(\n",
      "  File \"C:\\Users\\Danil\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\joblib\\externals\\loky\\backend\\context.py\", line 282, in _count_physical_cores\n",
      "    raise ValueError(f\"found {cpu_count_physical} physical cores < 1\")\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 456712 characters]",
      "text/plain": [
       "<Figure size 2500x3000 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Закодирование категориальных переменных\n",
    "df_encoded = pd.get_dummies(df_cleaned, drop_first=True)\n",
    "\n",
    "# Выбор подмножества данных для кластеризации\n",
    "features = df_encoded[['city_latitude']]\n",
    "\n",
    "# Масштабирование данных\n",
    "scaler = StandardScaler()\n",
    "scaled_features = scaler.fit_transform(features)\n",
    "\n",
    "# Кластеризация данных\n",
    "kmeans = KMeans(n_clusters=3)\n",
    "df_encoded['Cluster'] = kmeans.fit_predict(scaled_features)\n",
    "\n",
    "# Визуализация кластеров\n",
    "plt.figure(figsize=(25, 30))\n",
    "\n",
    "# Парный график 1: city_latitude vs state\n",
    "state_columns = [col for col in df_encoded.columns if col.startswith('state_')]\n",
    "if state_columns:\n",
    "    plt.subplot(4, 1, 1)\n",
    "    sns.scatterplot(x=df_encoded['city_latitude'], y=df_encoded[state_columns[0]], hue=df_encoded['Cluster'], palette='Set1', alpha=0.6)\n",
    "    plt.title('city_latitude vs state Clusters')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'state ({state_columns[0]})')\n",
    "else:\n",
    "    plt.subplot(4, 1, 1)\n",
    "    plt.text(0.5, 0.5, 'No state columns found', ha='center', va='center', fontsize=12)\n",
    "    plt.title('city_latitude vs state Clusters')\n",
    "\n",
    "# Парный график 2: city_latitude vs city\n",
    "city_columns = [col for col in df_encoded.columns if col.startswith('city_')]\n",
    "if city_columns:\n",
    "    plt.subplot(4, 1, 2)\n",
    "    sns.scatterplot(x=df_encoded['city_latitude'], y=df_encoded[city_columns[0]], hue=df_encoded['Cluster'], palette='Set1', alpha=0.6)\n",
    "    plt.title('city_latitude vs city Clusters')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'city ({city_columns[0]})')\n",
    "else:\n",
    "    plt.subplot(4, 1, 2)\n",
    "    plt.text(0.5, 0.5, 'No city columns found', ha='center', va='center', fontsize=12)\n",
    "    plt.title('city_latitude vs city Clusters')\n",
    "\n",
    "# Парный график 3: city_latitude vs state (другая категория)\n",
    "if len(state_columns) > 1:\n",
    "    plt.subplot(4, 1, 3)\n",
    "    sns.scatterplot(x=df_encoded['city_latitude'], y=df_encoded[state_columns[1]], hue=df_encoded['Cluster'], palette='Set1', alpha=0.6)\n",
    "    plt.title('city_latitude vs state Clusters')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'state ({state_columns[1]})')\n",
    "else:\n",
    "    plt.subplot(4, 1, 3)\n",
    "    plt.text(0.5, 0.5, 'Not enough state columns found', ha='center', va='center', fontsize=12)\n",
    "    plt.title('city_latitude vs state Clusters')\n",
    "\n",
    "# Парный график 4: city_latitude vs city (другая подкатегория)\n",
    "if len(city_columns) > 1:\n",
    "    plt.subplot(4, 1, 4)\n",
    "    sns.scatterplot(x=df_encoded['city_latitude'], y=df_encoded[city_columns[1]], hue=df_encoded['Cluster'], palette='Set1', alpha=0.6)\n",
    "    plt.title('city_latitude vs city Clusters')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'city ({city_columns[1]})')\n",
    "else:\n",
    "    plt.subplot(4, 1, 4)\n",
    "    plt.text(0.5, 0.5, 'Not enough city columns found', ha='center', va='center', fontsize=12)\n",
    "    plt.title('city_latitude vs city Clusters')\n",
    "\n",
    "# Настройка графиков\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KMeans (неиерархическая кластеризация) для сравнения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Центры кластеров:\n",
      " [[48.25734972]\n",
      " [32.43819138]\n",
      " [37.9835445 ]\n",
      " [43.07246158]]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 267176 characters]",
      "text/plain": [
       "<Figure size 1600x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Загрузка данных\n",
    "df = pd.read_csv(\"../../datasets/nuforc_reports.csv\")\n",
    "df = df.iloc[:1000].dropna()\n",
    "\n",
    "# Удаление несущественных столбцов\n",
    "columns_to_drop = ['summary', 'stats', 'report_link', 'posted', \"duration\"]\n",
    "df_cleaned = df.drop(columns=columns_to_drop)\n",
    "\n",
    "# Закодирование категориальных переменных\n",
    "df_encoded = pd.get_dummies(df_cleaned, drop_first=True)\n",
    "\n",
    "# Выбор подмножества данных для кластеризации\n",
    "features_used = ['city_latitude']\n",
    "data_to_scale = df_encoded[features_used]\n",
    "\n",
    "# Масштабирование данных\n",
    "scaler = StandardScaler()\n",
    "data_scaled = scaler.fit_transform(data_to_scale)\n",
    "\n",
    "# Кластеризация данных\n",
    "random_state = 42\n",
    "kmeans = KMeans(n_clusters=4, random_state=random_state)\n",
    "labels = kmeans.fit_predict(data_scaled)\n",
    "centers = kmeans.cluster_centers_\n",
    "\n",
    "# Отображение центроидов\n",
    "centers_original = scaler.inverse_transform(centers)  # Обратная стандартизация\n",
    "print(\"Центры кластеров:\\n\", centers_original)\n",
    "\n",
    "# Визуализация результатов кластеризации KMeans\n",
    "plt.figure(figsize=(16, 12))\n",
    "\n",
    "# Парный график 1: city_latitude vs state\n",
    "plt.subplot(2, 2, 1)\n",
    "state_columns = [col for col in df_encoded.columns if col.startswith('state_')]\n",
    "if state_columns:\n",
    "    sns.scatterplot(x=df_cleaned['city_latitude'], y=df_encoded[state_columns[0]], hue=labels, palette='Set1', alpha=0.6)\n",
    "    plt.title('KMeans Clustering: city_latitude vs state')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'state ({state_columns[0]})')\n",
    "else:\n",
    "    plt.title('KMeans Clustering: city_latitude vs state (No Data)')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel('state')\n",
    "\n",
    "# Парный график 2: city_latitude vs city\n",
    "plt.subplot(2, 2, 2)\n",
    "city_columns = [col for col in df_encoded.columns if col.startswith('city_')]\n",
    "if city_columns:\n",
    "    sns.scatterplot(x=df_cleaned['city_latitude'], y=df_encoded[city_columns[0]], hue=labels, palette='Set1', alpha=0.6)\n",
    "    plt.title('KMeans Clustering: city_latitude vs city')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'city ({city_columns[0]})')\n",
    "else:\n",
    "    plt.title('KMeans Clustering: city_latitude vs city (No Data)')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel('city')\n",
    "\n",
    "# Парный график 3: city_latitude vs state (другая категория)\n",
    "plt.subplot(2, 2, 3)\n",
    "if len(state_columns) > 1:\n",
    "    sns.scatterplot(x=df_cleaned['city_latitude'], y=df_encoded[state_columns[1]], hue=labels, palette='Set1', alpha=0.6)\n",
    "    plt.title('KMeans Clustering: city_latitude vs state')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'state ({state_columns[1]})')\n",
    "else:\n",
    "    plt.title('KMeans Clustering: city_latitude vs state (No Data)')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel('state')\n",
    "\n",
    "# Парный график 4: city_latitude vs city (другая подкатегория)\n",
    "plt.subplot(2, 2, 4)\n",
    "if len(city_columns) > 1:\n",
    "    sns.scatterplot(x=df_cleaned['city_latitude'], y=df_encoded[city_columns[1]], hue=labels, palette='Set1', alpha=0.6)\n",
    "    plt.title('KMeans Clustering: city_latitude vs city')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel(f'city ({city_columns[1]})')\n",
    "else:\n",
    "    plt.title('KMeans Clustering: city_latitude vs city (No Data)')\n",
    "    plt.xlabel('city_latitude')\n",
    "    plt.ylabel('city')\n",
    "\n",
    "# Настройка графиков\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA для визуализации сокращенной размерности"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 326408 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Применение PCA ТОЛЬКО к числовым данным\n",
    "pca = PCA(n_components=1)\n",
    "reduced_data = pca.fit_transform(data_scaled)\n",
    "\n",
    "# Визуализация сокращенных данных\n",
    "plt.figure(figsize=(16, 6))\n",
    "\n",
    "# График 1: PCA reduced data: Agglomerative Clustering\n",
    "plt.subplot(1, 2, 1)\n",
    "sns.scatterplot(x=range(len(reduced_data)), y=reduced_data[:, 0], hue=result, palette='Set1', alpha=0.6)\n",
    "plt.title('PCA reduced data: Agglomerative Clustering')\n",
    "plt.xlabel('Sample Index')\n",
    "plt.ylabel('Principal Component 1')\n",
    "\n",
    "# График 2: PCA reduced data: KMeans Clustering\n",
    "plt.subplot(1, 2, 2)\n",
    "sns.scatterplot(x=range(len(reduced_data)), y=reduced_data[:, 0], hue=labels, palette='Set1', alpha=0.6)\n",
    "plt.title('PCA reduced data: KMeans Clustering')\n",
    "plt.xlabel('Sample Index')\n",
    "plt.ylabel('Principal Component 1')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Анализ инерции для метода локтя (метод оценки суммы квадратов расстояний)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41708 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Список для хранения инерций\n",
    "inertias = []\n",
    "clusters_range = range(1, 11)\n",
    "\n",
    "# Вычисление инерции для каждого количества кластеров\n",
    "for i in clusters_range:\n",
    "    kmeans = KMeans(n_clusters=i, random_state=random_state)\n",
    "    kmeans.fit(data_scaled)\n",
    "    inertias.append(kmeans.inertia_)\n",
    "\n",
    "# Визуализация метода локтя\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(clusters_range, inertias, marker='o')\n",
    "plt.title('Метод локтя для оптимального k')\n",
    "plt.xlabel('Количество кластеров')\n",
    "plt.ylabel('Инерция')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Расчет коэффициентов силуэта"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 65956 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Список для хранения коэффициентов силуэта\n",
    "silhouette_scores = []\n",
    "\n",
    "# Вычисление коэффициентов силуэта для каждого количества кластеров\n",
    "for i in clusters_range[1:]:  \n",
    "    kmeans = KMeans(n_clusters=i, random_state=random_state)\n",
    "    labels = kmeans.fit_predict(data_scaled)\n",
    "    score = silhouette_score(data_scaled, labels)\n",
    "    silhouette_scores.append(score)\n",
    "\n",
    "# Построение диаграммы значений силуэта\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(clusters_range[1:], silhouette_scores, marker='o')\n",
    "plt.title('Коэффициенты силуэта для разных k')\n",
    "plt.xlabel('Количество кластеров')\n",
    "plt.ylabel('Коэффициент силуэта')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Средний коэффициент силуэта: 0.544\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 184564 characters]",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Добавляем индекс строки как дополнительный признак\n",
    "data_scaled_with_index = np.hstack((data_scaled, np.arange(data_scaled.shape[0]).reshape(-1, 1)))\n",
    "\n",
    "# ========================\n",
    "# Применение K-Means\n",
    "# ========================\n",
    "kmeans = KMeans(n_clusters=3, random_state=42)  \n",
    "df_clusters = kmeans.fit_predict(data_scaled)\n",
    "\n",
    "# ========================\n",
    "# Оценка качества кластеризации\n",
    "# ========================\n",
    "silhouette_avg = silhouette_score(data_scaled, df_clusters)\n",
    "print(f'Средний коэффициент силуэта: {silhouette_avg:.3f}')\n",
    "\n",
    "# ========================\n",
    "# Визуализация кластеров\n",
    "# ========================\n",
    "pca = PCA(n_components=2)\n",
    "df_pca = pca.fit_transform(data_scaled_with_index)\n",
    "\n",
    "plt.figure(figsize=(10, 7))\n",
    "sns.scatterplot(x=df_pca[:, 0], y=df_pca[:, 1], hue=df_clusters, palette='viridis', alpha=0.7)\n",
    "plt.title('Визуализация кластеров с помощью K-Means')\n",
    "plt.xlabel('Первая компонента PCA')\n",
    "plt.ylabel('Вторая компонента PCA')\n",
    "plt.legend(title='Кластер', loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Средний коэффициент силуэта, равный 0.544, указывает на хорошую кластеризацию.  \n",
    "\n",
    "Средний коэффициент силуэта (silhouette score) указывает на качество кластеризации, измеряя, насколько хорошо точки внутри одного кластера близки друг к другу по сравнению с точками из других кластеров. Значения коэффициента силуэта находятся в диапазоне от -1 до 1:\n",
    "\n",
    "1: Указывает на идеально плотные и четко разделенные кластеры.  \n",
    "0: Указывает на перекрытие кластеров или слабую структуру кластеризации.  \n",
    "Отрицательные значения: Указывают, что точки в кластере расположены ближе к другому кластеру, чем к своему."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Средний коэффициент силуэта (агломеративная кластеризация): 0.542\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 192440 characters]",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "\n",
    "# Добавляем индекс строки как дополнительный признак\n",
    "data_scaled_with_index = np.hstack((data_scaled, np.arange(data_scaled.shape[0]).reshape(-1, 1)))\n",
    "\n",
    "# ========================\n",
    "# Агломеративная кластеризация\n",
    "# ========================\n",
    "agg_cluster = AgglomerativeClustering(n_clusters=3)  \n",
    "labels_agg = agg_cluster.fit_predict(data_scaled)\n",
    "\n",
    "# ========================\n",
    "# Оценка качества кластеризации\n",
    "# ========================\n",
    "silhouette_avg_agg = silhouette_score(data_scaled, labels_agg)\n",
    "print(f'Средний коэффициент силуэта (агломеративная кластеризация): {silhouette_avg_agg:.3f}')\n",
    "\n",
    "# ========================\n",
    "# Визуализация кластеров\n",
    "# ========================\n",
    "pca = PCA(n_components=2)\n",
    "df_pca = pca.fit_transform(data_scaled_with_index)\n",
    "\n",
    "plt.figure(figsize=(10, 7))\n",
    "sns.scatterplot(x=df_pca[:, 0], y=df_pca[:, 1], hue=labels_agg, palette='viridis', alpha=0.7)\n",
    "plt.title('Визуализация кластеров с помощью агломеративной кластеризации')\n",
    "plt.xlabel('Первая компонента PCA')\n",
    "plt.ylabel('Вторая компонента PCA')\n",
    "plt.legend(title='Кластер', loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Значение коэффициента силуэта лежит в диапазоне от -1 до 1. Ближе к 1: Хорошо сформированные, плотные кластеры, четко отделенные друг от друга.  \n",
    "\n",
    "Ближе к 0: Кластеры пересекаются или слабо разделены, не имеют четких границ. Точки расположены одинаково близко как к своему кластеру, так и к соседним. \n",
    "Ближе к -1 (Отрицательные значения): Некоторые точки скорее относятся к другим кластерам, чем к текущему (ближе к центрам других кластеров). Очень плохая кластеризация.  \n",
    "Ближе к 1: Все точки внутри каждого кластера плотно сгруппированы и значительно удалены от точек других кластеров. Свидетельствует о четкой и хорошо разделенной структуре данных. Единица говорит об идеальной кластеризации.\n",
    "\n",
    "Средний коэффициент силуэта, равный 0.542, указывает на то, что кластеры имеют хорошее разделение и четкие границы. Точки внутри каждого кластера достаточно плотно сгруппированы и значительно удалены от точек других кластеров, что свидетельствует о четкой и хорошо разделенной структуре данных."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
