{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.kaggle.com/datasets/harishkumardatalab/medical-insurance-price-prediction Набор представляет собой данные о мед страховке.\n",
    "Пример цели: Рассчитать стоимость будущей страховки\n",
    "Входные данные: возраст, пол, индекс массы тела, дети, курит ли, регион, стоимость"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['age', 'sex', 'bmi', 'children', 'smoker', 'region', 'charges'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "df = pd.read_csv(\".//static//csv//Medical_insurance.csv\")\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12888 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 62744 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Визуализация данных - ящик с усами. Как видим - у выборки есть смещение в меньшую сторону\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.boxplot(x=df[\"bmi\"])\n",
    "plt.title(\"Box Plot для bmi\")\n",
    "plt.xlabel(\"bmi\")\n",
    "plt.show()\n",
    "\n",
    "#Визуализируем отношение стоимости мед счетов и возраста\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.scatter(df[\"age\"], df[\"charges\"])\n",
    "plt.xlabel('Age')\n",
    "plt.ylabel(\"Charge\")\n",
    "plt.title('Scatter Plot of Age vs Charge')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Выбросы:\n",
      "      age     sex    bmi  children smoker     region     charges\n",
      "14     27    male  42.13         0    yes  southeast  39611.7577\n",
      "19     30    male  35.30         0    yes  southwest  36837.4670\n",
      "23     34  female  31.92         1    yes  northeast  37701.8768\n",
      "29     31    male  36.30         2    yes  southwest  38711.0000\n",
      "30     22    male  35.60         0    yes  southwest  35585.5760\n",
      "...   ...     ...    ...       ...    ...        ...         ...\n",
      "2735   52    male  41.80         2    yes  southeast  47269.8540\n",
      "2736   64    male  36.96         2    yes  southeast  49577.6624\n",
      "2744   32    male  33.63         1    yes  northeast  37607.5277\n",
      "2764   22  female  31.02         3    yes  southeast  35595.5898\n",
      "2765   47    male  36.08         1    yes  southeast  42211.1382\n",
      "\n",
      "[296 rows x 7 columns]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 62880 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# Статистический анализ для определения выбросов\n",
    "Q1 = df[\"charges\"].quantile(0.25)\n",
    "Q3 = df[\"charges\"].quantile(0.75)\n",
    "IQR = Q3 - Q1\n",
    "\n",
    "# Определение порога для выбросов\n",
    "threshold = 1.5 * IQR\n",
    "outliers = (df[\"charges\"] < (Q1 - threshold)) | (df[\"charges\"] > (Q3 + threshold))\n",
    "\n",
    "# Вывод выбросов\n",
    "print(\"Выбросы:\")\n",
    "print(df[outliers])\n",
    "\n",
    "# Обработка выбросов\n",
    "# В данном случае мы обнулим выбросы\n",
    "median_charge = df[\"charges\"].median()\n",
    "df.loc[outliers, \"charges\"] = 0\n",
    "df = df[df.charges != 0]\n",
    "\n",
    "# Визуализация данных после обработки\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.scatter(df[\"age\"], df[\"charges\"])\n",
    "plt.xlabel(\"Age\")\n",
    "plt.ylabel(\"Charge\")\n",
    "plt.title(\"Scatter Plot of Age vs Charge\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь создадим выборки."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер обучающей выборки: 1485\n",
      "Размер контрольной выборки: 495\n",
      "Размер тестовой выборки: 496\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "train_df, temp_df = train_test_split(df, test_size=0.4, random_state=42)\n",
    "\n",
    "# Разделение остатка на контрольную и тестовую выборки\n",
    "val_df, test_df = train_test_split(temp_df, test_size=0.5, random_state=42)\n",
    "\n",
    "# Проверка размеров выборок\n",
    "print(\"Размер обучающей выборки:\", len(train_df))\n",
    "print(\"Размер контрольной выборки:\", len(val_df))\n",
    "print(\"Размер тестовой выборки:\", len(test_df))\n",
    "\n",
    "# Сохранение выборок в файлы\n",
    "train_df.to_csv(\".//static//csv//train_data.csv\", index=False)\n",
    "val_df.to_csv(\".//static//csv//val_data.csv\", index=False)\n",
    "test_df.to_csv(\".//static//csv//test_data.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Проанализируем сбалансированность выборок"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Распределение Review_type в обучающей выборке:\n",
      "sex\n",
      "female    765\n",
      "male      720\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 48.48%\n",
      "Процент женщин: 51.52%\n",
      "\n",
      "Распределение Review_type в контрольной выборке:\n",
      "sex\n",
      "male      257\n",
      "female    238\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 51.92%\n",
      "Процент женщин: 48.08%\n",
      "\n",
      "Распределение Review_type в тестовой выборке:\n",
      "sex\n",
      "female    259\n",
      "male      237\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 47.78%\n",
      "Процент женщин: 52.22%\n",
      "\n",
      "Аугментация данных не требуется.\n",
      "Аугментация данных не требуется.\n",
      "Аугментация данных не требуется.\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(\".//static//csv//train_data.csv\")\n",
    "val_df = pd.read_csv(\".//static//csv//val_data.csv\")\n",
    "test_df = pd.read_csv(\".//static//csv//test_data.csv\")\n",
    "\n",
    "# Оценка сбалансированности\n",
    "def check_balance(df, name):\n",
    "    counts = df['sex'].value_counts()\n",
    "    print(f\"Распределение Review_type в {name}:\")\n",
    "    print(counts)\n",
    "    print(f\"Процент мужчин: {counts['male'] / len(df) * 100:.2f}%\")\n",
    "    print(f\"Процент женщин: {counts['female'] / len(df) * 100:.2f}%\")\n",
    "    print()\n",
    "\n",
    "# Определение необходимости аугментации данных\n",
    "def need_augmentation(df):\n",
    "    counts = df['sex'].value_counts()\n",
    "    ratio = counts['male'] / counts['female']\n",
    "    if ratio > 1.5 or ratio < 0.67:\n",
    "        print(\"Необходима аугментация данных для балансировки классов.\")\n",
    "    else:\n",
    "        print(\"Аугментация данных не требуется.\")\n",
    "        \n",
    "check_balance(train_df, \"обучающей выборке\")\n",
    "check_balance(val_df, \"контрольной выборке\")\n",
    "check_balance(test_df, \"тестовой выборке\")\n",
    "\n",
    "need_augmentation(train_df)\n",
    "need_augmentation(val_df)\n",
    "need_augmentation(test_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "По результатам анализа требуется приращение, соотношения вне допустимого диапазона"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Оверсэмплинг:\n",
      "Распределение sex в обучающей выборке:\n",
      "sex\n",
      "male      765\n",
      "female    765\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 50.00%\n",
      "Процент женщин: 50.00%\n",
      "\n",
      "Распределение sex в контрольной выборке:\n",
      "sex\n",
      "male      257\n",
      "female    257\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 50.00%\n",
      "Процент женщин: 50.00%\n",
      "\n",
      "Распределение sex в тестовой выборке:\n",
      "sex\n",
      "female    259\n",
      "male      259\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 50.00%\n",
      "Процент женщин: 50.00%\n",
      "\n",
      "Андерсэмплинг:\n",
      "Распределение sex в обучающей выборке:\n",
      "sex\n",
      "female    720\n",
      "male      720\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 50.00%\n",
      "Процент женщин: 50.00%\n",
      "\n",
      "Распределение sex в контрольной выборке:\n",
      "sex\n",
      "female    238\n",
      "male      238\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 50.00%\n",
      "Процент женщин: 50.00%\n",
      "\n",
      "Распределение sex в тестовой выборке:\n",
      "sex\n",
      "female    237\n",
      "male      237\n",
      "Name: count, dtype: int64\n",
      "Процент мужчин: 50.00%\n",
      "Процент женщин: 50.00%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# Загрузка данных\n",
    "train_df = pd.read_csv(\".//static//csv//train_data.csv\")\n",
    "val_df = pd.read_csv(\".//static//csv//val_data.csv\")\n",
    "test_df = pd.read_csv(\".//static//csv//test_data.csv\")\n",
    "\n",
    "# Преобразование категориальных признаков в числовые\n",
    "def encode(df):\n",
    "    label_encoders = {}\n",
    "    for column in df.select_dtypes(include=['object']).columns:\n",
    "        if column != 'sex':  # Пропускаем целевую переменную\n",
    "            le = LabelEncoder()\n",
    "            df[column] = le.fit_transform(df[column])\n",
    "            label_encoders[column] = le\n",
    "    return label_encoders\n",
    "\n",
    "# Преобразование целевой переменной в числовые значения\n",
    "def encode_target(df):\n",
    "    le = LabelEncoder()\n",
    "    df['sex'] = le.fit_transform(df['sex'])\n",
    "    return le\n",
    "\n",
    "# Применение кодирования\n",
    "label_encoders = encode(train_df)\n",
    "encode(val_df)\n",
    "encode(test_df)\n",
    "\n",
    "# Кодирование целевой переменной\n",
    "le_target = encode_target(train_df)\n",
    "encode_target(val_df)\n",
    "encode_target(test_df)\n",
    "\n",
    "# Проверка типов данных\n",
    "def check_data_types(df):\n",
    "    for column in df.columns:\n",
    "        if df[column].dtype == 'object':\n",
    "            print(f\"Столбец '{column}' содержит строковые данные.\")\n",
    "\n",
    "check_data_types(train_df)\n",
    "check_data_types(val_df)\n",
    "check_data_types(test_df)\n",
    "\n",
    "# Функция для выполнения oversampling\n",
    "def oversample(df):\n",
    "    if 'sex' not in df.columns:\n",
    "        print(\"Столбец 'sex' отсутствует.\")\n",
    "        return df\n",
    "    \n",
    "    X = df.drop('sex', axis=1)\n",
    "    y = df['sex']\n",
    "    \n",
    "    oversampler = RandomOverSampler(random_state=42)\n",
    "    X_resampled, y_resampled = oversampler.fit_resample(X, y) # type: ignore\n",
    "    \n",
    "    resampled_df = pd.concat([X_resampled, y_resampled], axis=1)\n",
    "    return resampled_df\n",
    "\n",
    "# Функция для выполнения undersampling\n",
    "def undersample(df):\n",
    "    if 'sex' not in df.columns:\n",
    "        print(\"Столбец 'sex' отсутствует.\")\n",
    "        return df\n",
    "    \n",
    "    X = df.drop('sex', axis=1)\n",
    "    y = df['sex']\n",
    "    \n",
    "    undersampler = RandomUnderSampler(random_state=42)\n",
    "    X_resampled, y_resampled = undersampler.fit_resample(X, y) # type: ignore\n",
    "    \n",
    "    resampled_df = pd.concat([X_resampled, y_resampled], axis=1)\n",
    "    return resampled_df\n",
    "\n",
    "# Применение oversampling и undersampling к каждой выборке\n",
    "train_df_oversampled = oversample(train_df)\n",
    "val_df_oversampled = oversample(val_df)\n",
    "test_df_oversampled = oversample(test_df)\n",
    "\n",
    "train_df_undersampled = undersample(train_df)\n",
    "val_df_undersampled = undersample(val_df)\n",
    "test_df_undersampled = undersample(test_df)\n",
    "\n",
    "# Обратное преобразование целевой переменной в строковые метки\n",
    "def decode_target(df, le_target):\n",
    "    df['sex'] = le_target.inverse_transform(df['sex'])\n",
    "\n",
    "decode_target(train_df_oversampled, le_target)\n",
    "decode_target(val_df_oversampled, le_target)\n",
    "decode_target(test_df_oversampled, le_target)\n",
    "\n",
    "decode_target(train_df_undersampled, le_target)\n",
    "decode_target(val_df_undersampled, le_target)\n",
    "decode_target(test_df_undersampled, le_target)\n",
    "\n",
    "# Проверка результатов\n",
    "def check_balance(df, name):\n",
    "    if 'sex' not in df.columns:\n",
    "        print(f\"Столбец 'sex' отсутствует в {name}.\")\n",
    "        return\n",
    "    \n",
    "    counts = df['sex'].value_counts()\n",
    "    print(f\"Распределение sex в {name}:\")\n",
    "    print(counts)\n",
    "    \n",
    "    if 'male' in counts and 'female' in counts:\n",
    "        print(f\"Процент мужчин: {counts['male'] / len(df) * 100:.2f}%\")\n",
    "        print(f\"Процент женщин: {counts['female'] / len(df) * 100:.2f}%\")\n",
    "    else:\n",
    "        print(\"Отсутствуют один или оба класса (male/female.\")\n",
    "    print()\n",
    "\n",
    "# Проверка сбалансированности после oversampling\n",
    "print(\"Оверсэмплинг:\")\n",
    "check_balance(train_df_oversampled, \"обучающей выборке\")\n",
    "check_balance(val_df_oversampled, \"контрольной выборке\")\n",
    "check_balance(test_df_oversampled, \"тестовой выборке\")\n",
    "\n",
    "# Проверка сбалансированности после undersampling\n",
    "print(\"Андерсэмплинг:\")\n",
    "check_balance(train_df_undersampled, \"обучающей выборке\")\n",
    "check_balance(val_df_undersampled, \"контрольной выборке\")\n",
    "check_balance(test_df_undersampled, \"тестовой выборке\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stroke Prediction Dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.kaggle.com/datasets/fedesoriano/stroke-prediction-dataset Датасет инсультов\n",
    "Цель: выжить\n",
    "Входные данные: пол, возраст, женат ли, есть ли заболевания сердца"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         id  gender  age  hypertension  heart_disease ever_married  \\\n",
      "0      9046    Male   67             0              1          Yes   \n",
      "1     51676  Female   61             0              0          Yes   \n",
      "2     31112    Male   80             0              1          Yes   \n",
      "3     60182  Female   49             0              0          Yes   \n",
      "4      1665  Female   79             1              0          Yes   \n",
      "...     ...     ...  ...           ...            ...          ...   \n",
      "5105  18234  Female   80             1              0          Yes   \n",
      "5106  44873  Female   81             0              0          Yes   \n",
      "5107  19723  Female   35             0              0          Yes   \n",
      "5108  37544    Male   51             0              0          Yes   \n",
      "5109  44679  Female   44             0              0          Yes   \n",
      "\n",
      "          work_type Residence_type  avg_glucose_level   bmi   smoking_status  \\\n",
      "0           Private          Urban             228.69  36.6  formerly smoked   \n",
      "1     Self-employed          Rural             202.21   NaN     never smoked   \n",
      "2           Private          Rural             105.92  32.5     never smoked   \n",
      "3           Private          Urban             171.23  34.4           smokes   \n",
      "4     Self-employed          Rural             174.12  24.0     never smoked   \n",
      "...             ...            ...                ...   ...              ...   \n",
      "5105        Private          Urban              83.75   NaN     never smoked   \n",
      "5106  Self-employed          Urban             125.20  40.0     never smoked   \n",
      "5107  Self-employed          Rural              82.99  30.6     never smoked   \n",
      "5108        Private          Rural             166.29  25.6  formerly smoked   \n",
      "5109       Govt_job          Urban              85.28  26.2          Unknown   \n",
      "\n",
      "      stroke  \n",
      "0          1  \n",
      "1          1  \n",
      "2          1  \n",
      "3          1  \n",
      "4          1  \n",
      "...      ...  \n",
      "5105       0  \n",
      "5106       0  \n",
      "5107       0  \n",
      "5108       0  \n",
      "5109       0  \n",
      "\n",
      "[5110 rows x 12 columns]\n",
      "Index(['id', 'gender', 'age', 'hypertension', 'heart_disease', 'ever_married',\n",
      "       'work_type', 'Residence_type', 'avg_glucose_level', 'bmi',\n",
      "       'smoking_status', 'stroke'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "df = pd.read_csv(\".//static//csv//healthcare-dataset-stroke-data.csv\", sep=\",\")\n",
    "\n",
    "df[\"age\"] = df[\"age\"].astype(int)\n",
    "print(df)\n",
    "df[\"age\"].dtype\n",
    "\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness', 'Insulin',\n",
      "       'BMI', 'DiabetesPedigreeFunction', 'Age', 'Outcome'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns\n",
    "df = pd.read_csv(\".//static//csv//diabetes.csv\")\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Анализируем датафрейм при помощи \"ящика с усами\". Естьсмещение в сторону меньших значений, это можно исправить при помощи oversampling и undersampling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14720 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Box plot для столбца возраст\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.boxplot(x=df['Age'])\n",
    "plt.title('Box Plot для Age')\n",
    "plt.xlabel('Age')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 61124 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Выбросы:\n",
      "     Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
      "4              0      137             40             35      168  43.1   \n",
      "12            10      139             80              0        0  27.1   \n",
      "39             4      111             72             47      207  37.1   \n",
      "45             0      180             66             39        0  42.0   \n",
      "58             0      146             82              0        0  40.5   \n",
      "100            1      163             72              0        0  39.0   \n",
      "147            2      106             64             35      119  30.5   \n",
      "187            1      128             98             41       58  32.0   \n",
      "218            5       85             74             22        0  29.0   \n",
      "228            4      197             70             39      744  36.7   \n",
      "243            6      119             50             22      176  27.1   \n",
      "245            9      184             85             15        0  30.0   \n",
      "259           11      155             76             28      150  33.3   \n",
      "292            2      128             78             37      182  43.3   \n",
      "308            0      128             68             19      180  30.5   \n",
      "330            8      118             72             19        0  23.1   \n",
      "370            3      173             82             48      465  38.4   \n",
      "371            0      118             64             23       89   0.0   \n",
      "383            1       90             62             18       59  25.1   \n",
      "395            2      127             58             24      275  27.7   \n",
      "445            0      180             78             63       14  59.4   \n",
      "534            1       77             56             30       56  33.3   \n",
      "593            2       82             52             22      115  28.5   \n",
      "606            1      181             78             42      293  40.0   \n",
      "618            9      112             82             24        0  28.2   \n",
      "621            2       92             76             20        0  24.2   \n",
      "622            6      183             94              0        0  40.8   \n",
      "659            3       80             82             31       70  34.2   \n",
      "661            1      199             76             43        0  42.9   \n",
      "\n",
      "     DiabetesPedigreeFunction  Age  Outcome  \n",
      "4                       2.288   33        1  \n",
      "12                      1.441   57        0  \n",
      "39                      1.390   56        1  \n",
      "45                      1.893   25        1  \n",
      "58                      1.781   44        0  \n",
      "100                     1.222   33        1  \n",
      "147                     1.400   34        0  \n",
      "187                     1.321   33        1  \n",
      "218                     1.224   32        1  \n",
      "228                     2.329   31        0  \n",
      "243                     1.318   33        1  \n",
      "245                     1.213   49        1  \n",
      "259                     1.353   51        1  \n",
      "292                     1.224   31        1  \n",
      "308                     1.391   25        1  \n",
      "330                     1.476   46        0  \n",
      "370                     2.137   25        1  \n",
      "371                     1.731   21        0  \n",
      "383                     1.268   25        0  \n",
      "395                     1.600   25        0  \n",
      "445                     2.420   25        1  \n",
      "534                     1.251   24        0  \n",
      "593                     1.699   25        0  \n",
      "606                     1.258   22        1  \n",
      "618                     1.282   50        1  \n",
      "621                     1.698   28        0  \n",
      "622                     1.461   45        0  \n",
      "659                     1.292   27        1  \n",
      "661                     1.394   22        1  \n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 66876 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Имеется смещение в меньшую сторону\n",
    "df_cleaned = df.dropna()\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.scatter(df[\"Age\"], df[\"DiabetesPedigreeFunction\"])\n",
    "plt.xlabel(\"Age\")\n",
    "plt.ylabel(\"DiabetesPedigreeFunction\")\n",
    "plt.title(\"Scatter Plot of Age vs DiabetesPedigreeFunction\")\n",
    "plt.show()\n",
    "\n",
    "# Статистический анализ для определения выбросов\n",
    "Q1 = df[\"DiabetesPedigreeFunction\"].quantile(0.25)\n",
    "Q3 = df[\"DiabetesPedigreeFunction\"].quantile(0.75)\n",
    "IQR = Q3 - Q1\n",
    "\n",
    "# Определение порога для выбросов\n",
    "threshold = 1.5 * IQR\n",
    "outliers = (df[\"DiabetesPedigreeFunction\"] < (Q1 - threshold)) | (\n",
    "    df[\"DiabetesPedigreeFunction\"] > (Q3 + threshold)\n",
    ")\n",
    "\n",
    "# Вывод выбросов\n",
    "print(\"Выбросы:\")\n",
    "print(df[outliers])\n",
    "\n",
    "# Обработка выбросов\n",
    "# В данном случае мы уберем выбросы\n",
    "median_charge = df[\"DiabetesPedigreeFunction\"].median()\n",
    "df.loc[outliers, \"DiabetesPedigreeFunction\"] = 0\n",
    "df = df[df.DiabetesPedigreeFunction != 0]\n",
    "\n",
    "# Визуализация данных после обработки\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.scatter(df[\"Age\"], df[\"DiabetesPedigreeFunction\"])\n",
    "plt.xlabel(\"Age\")\n",
    "plt.ylabel(\"DiabetesPedigreeFunction\")\n",
    "plt.title(\"Scatter Plot of Age vs DiabetesPedigreeFunction\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Разбиение набора данных на обучающую, контрольную и тестовую выборки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер обучающей выборки: 460\n",
      "Размер контрольной выборки: 154\n",
      "Размер тестовой выборки: 154\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Разделение на обучающую и тестовую выборки\n",
    "train_df, test_df = train_test_split(df_cleaned, test_size=0.2, random_state=42)\n",
    "\n",
    "# Разделение обучающей выборки на обучающую и контрольную\n",
    "train_df, val_df = train_test_split(train_df, test_size=0.25, random_state=42)\n",
    "\n",
    "print(\"Размер обучающей выборки:\", len(train_df))\n",
    "print(\"Размер контрольной выборки:\", len(val_df))\n",
    "print(\"Размер тестовой выборки:\", len(test_df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Видим недостаток баланса:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Распределение Age в обучающей выборке:\n",
      "Age\n",
      "22    39\n",
      "21    36\n",
      "25    31\n",
      "24    25\n",
      "26    23\n",
      "27    22\n",
      "28    21\n",
      "23    21\n",
      "41    17\n",
      "31    15\n",
      "29    15\n",
      "37    14\n",
      "30    12\n",
      "45    11\n",
      "46    11\n",
      "36    11\n",
      "42    10\n",
      "34    10\n",
      "38    10\n",
      "32     9\n",
      "35     8\n",
      "52     8\n",
      "33     7\n",
      "43     7\n",
      "40     7\n",
      "47     5\n",
      "50     5\n",
      "51     5\n",
      "39     4\n",
      "48     4\n",
      "57     4\n",
      "66     3\n",
      "49     3\n",
      "44     3\n",
      "58     2\n",
      "55     2\n",
      "53     2\n",
      "63     2\n",
      "67     2\n",
      "54     2\n",
      "56     2\n",
      "60     2\n",
      "59     2\n",
      "61     1\n",
      "65     1\n",
      "81     1\n",
      "69     1\n",
      "68     1\n",
      "70     1\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Распределение Age в контрольной выборке:\n",
      "Age\n",
      "22    20\n",
      "21    15\n",
      "24    11\n",
      "23    10\n",
      "25     9\n",
      "33     8\n",
      "26     7\n",
      "31     6\n",
      "30     5\n",
      "28     5\n",
      "27     5\n",
      "39     4\n",
      "29     4\n",
      "42     4\n",
      "40     4\n",
      "32     3\n",
      "37     2\n",
      "46     2\n",
      "51     2\n",
      "44     2\n",
      "34     2\n",
      "43     2\n",
      "41     2\n",
      "45     2\n",
      "54     2\n",
      "35     2\n",
      "58     1\n",
      "53     1\n",
      "61     1\n",
      "59     1\n",
      "64     1\n",
      "36     1\n",
      "50     1\n",
      "72     1\n",
      "49     1\n",
      "47     1\n",
      "66     1\n",
      "62     1\n",
      "69     1\n",
      "55     1\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Распределение Age в тестовой выборке:\n",
      "Age\n",
      "22    13\n",
      "21    12\n",
      "24    10\n",
      "29    10\n",
      "28     9\n",
      "25     8\n",
      "23     7\n",
      "38     6\n",
      "27     5\n",
      "39     4\n",
      "30     4\n",
      "43     4\n",
      "42     4\n",
      "32     4\n",
      "36     4\n",
      "58     4\n",
      "31     3\n",
      "62     3\n",
      "44     3\n",
      "26     3\n",
      "41     3\n",
      "37     3\n",
      "60     3\n",
      "54     2\n",
      "50     2\n",
      "34     2\n",
      "65     2\n",
      "63     2\n",
      "45     2\n",
      "40     2\n",
      "33     2\n",
      "53     2\n",
      "55     1\n",
      "48     1\n",
      "67     1\n",
      "56     1\n",
      "51     1\n",
      "49     1\n",
      "57     1\n",
      "Name: count, dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def check_balance(df, name):\n",
    "    counts = df['Age'].value_counts()\n",
    "    print(f\"Распределение Age в {name}:\")\n",
    "    print(counts)\n",
    "    print()\n",
    "\n",
    "check_balance(train_df, \"обучающей выборке\")\n",
    "check_balance(val_df, \"контрольной выборке\")\n",
    "check_balance(test_df, \"тестовой выборке\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Используем oversample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Распределение Age в обучающей выборке после oversampling:\n",
      "Age\n",
      "26    39\n",
      "25    39\n",
      "33    39\n",
      "23    39\n",
      "35    39\n",
      "39    39\n",
      "29    39\n",
      "27    39\n",
      "21    39\n",
      "28    39\n",
      "41    39\n",
      "31    39\n",
      "24    39\n",
      "22    39\n",
      "42    39\n",
      "32    39\n",
      "36    39\n",
      "51    39\n",
      "34    39\n",
      "30    39\n",
      "45    39\n",
      "49    39\n",
      "57    39\n",
      "53    39\n",
      "48    39\n",
      "55    39\n",
      "46    39\n",
      "47    39\n",
      "52    39\n",
      "50    39\n",
      "37    39\n",
      "58    39\n",
      "61    39\n",
      "40    39\n",
      "65    39\n",
      "67    39\n",
      "38    39\n",
      "66    39\n",
      "81    39\n",
      "44    39\n",
      "43    39\n",
      "56    39\n",
      "54    39\n",
      "60    39\n",
      "69    39\n",
      "70    39\n",
      "68    39\n",
      "63    39\n",
      "59    39\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Распределение Age в контрольной выборке после oversampling:\n",
      "Age\n",
      "25    20\n",
      "58    20\n",
      "27    20\n",
      "26    20\n",
      "31    20\n",
      "22    20\n",
      "28    20\n",
      "39    20\n",
      "21    20\n",
      "45    20\n",
      "41    20\n",
      "61    20\n",
      "51    20\n",
      "64    20\n",
      "36    20\n",
      "30    20\n",
      "53    20\n",
      "40    20\n",
      "23    20\n",
      "59    20\n",
      "29    20\n",
      "42    20\n",
      "46    20\n",
      "43    20\n",
      "44    20\n",
      "34    20\n",
      "24    20\n",
      "32    20\n",
      "33    20\n",
      "72    20\n",
      "35    20\n",
      "50    20\n",
      "54    20\n",
      "47    20\n",
      "49    20\n",
      "66    20\n",
      "62    20\n",
      "69    20\n",
      "55    20\n",
      "37    20\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Распределение Age в тестовой выборке после oversampling:\n",
      "Age\n",
      "43    13\n",
      "21    13\n",
      "34    13\n",
      "50    13\n",
      "55    13\n",
      "22    13\n",
      "44    13\n",
      "37    13\n",
      "65    13\n",
      "40    13\n",
      "60    13\n",
      "29    13\n",
      "28    13\n",
      "24    13\n",
      "36    13\n",
      "48    13\n",
      "45    13\n",
      "39    13\n",
      "41    13\n",
      "27    13\n",
      "23    13\n",
      "38    13\n",
      "25    13\n",
      "67    13\n",
      "31    13\n",
      "30    13\n",
      "54    13\n",
      "33    13\n",
      "62    13\n",
      "26    13\n",
      "56    13\n",
      "58    13\n",
      "53    13\n",
      "63    13\n",
      "42    13\n",
      "32    13\n",
      "51    13\n",
      "49    13\n",
      "57    13\n",
      "Name: count, dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "def oversample(df):\n",
    "    X = df.drop('Age', axis=1)\n",
    "    y = df['Age']\n",
    "    \n",
    "    oversampler = RandomOverSampler(random_state=42)\n",
    "    X_resampled, y_resampled = oversampler.fit_resample(X, y) # type: ignore\n",
    "    \n",
    "    resampled_df = pd.concat([X_resampled, y_resampled], axis=1)\n",
    "    return resampled_df\n",
    "\n",
    "train_df_oversampled = oversample(train_df)\n",
    "val_df_oversampled = oversample(val_df)\n",
    "test_df_oversampled = oversample(test_df)\n",
    "\n",
    "check_balance(train_df_oversampled, \"обучающей выборке после oversampling\")\n",
    "check_balance(val_df_oversampled, \"контрольной выборке после oversampling\")\n",
    "check_balance(test_df_oversampled, \"тестовой выборке после oversampling\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diamonds Prices2022"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В данном наборе данных представлена цена на алмазы. Входные данные: цвет, караты, цена, чистота, размеры\n",
    "Цель: узнать, какие алмазы ценятся больше всего"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Unnamed: 0', 'carat', 'cut', 'color', 'clarity', 'depth', 'table',\n",
      "       'price', 'x', 'y', 'z'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "df = pd.read_csv(\".//static//csv//Diamonds Prices2022.csv\")\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 15276 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.boxplot(x=df['carat'])\n",
    "plt.title('Box Plot для carat')\n",
    "plt.xlabel('carat')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 68688 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "# Визуализация данных после обработки\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.scatter(df[\"carat\"], df[\"price\"])\n",
    "plt.xlabel(\"carat\")\n",
    "plt.ylabel(\"price\")\n",
    "plt.title(\"Scatter Plot of Price vs Carat\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Удаление строк с пустыми значениями"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cleaned = df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Разбиение набора данных на обучающую, контрольную и тестовую выборки"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Применение методов приращения данных (аугментации)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер обучающей выборки: 32365\n",
      "Размер контрольной выборки: 10789\n",
      "Размер тестовой выборки: 10789\n",
      "Распределение price в обучающей выборке:\n",
      "price\n",
      "789      80\n",
      "605      79\n",
      "544      72\n",
      "552      72\n",
      "828      71\n",
      "         ..\n",
      "9942      1\n",
      "7787      1\n",
      "18663     1\n",
      "7979      1\n",
      "8164      1\n",
      "Name: count, Length: 9496, dtype: int64\n",
      "\n",
      "Распределение price в контрольной выборке:\n",
      "price\n",
      "625      34\n",
      "828      31\n",
      "605      30\n",
      "789      26\n",
      "544      26\n",
      "         ..\n",
      "4188      1\n",
      "7541      1\n",
      "3498      1\n",
      "3314      1\n",
      "12196     1\n",
      "Name: count, Length: 5383, dtype: int64\n",
      "\n",
      "Распределение price в тестовой выборке:\n",
      "price\n",
      "802     33\n",
      "844     29\n",
      "776     29\n",
      "675     26\n",
      "645     25\n",
      "        ..\n",
      "1567     1\n",
      "5529     1\n",
      "2031     1\n",
      "417      1\n",
      "5431     1\n",
      "Name: count, Length: 5338, dtype: int64\n",
      "\n",
      "Распределение price в обучающей выборке после oversampling:\n",
      "price\n",
      "5076    80\n",
      "1789    80\n",
      "3931    80\n",
      "1263    80\n",
      "2026    80\n",
      "        ..\n",
      "3678    80\n",
      "4592    80\n",
      "516     80\n",
      "7152    80\n",
      "2353    80\n",
      "Name: count, Length: 9496, dtype: int64\n",
      "\n",
      "Распределение price в контрольной выборке после oversampling:\n",
      "price\n",
      "966      34\n",
      "13638    34\n",
      "3669     34\n",
      "1052     34\n",
      "2818     34\n",
      "         ..\n",
      "4032     34\n",
      "544      34\n",
      "3362     34\n",
      "6559     34\n",
      "792      34\n",
      "Name: count, Length: 5383, dtype: int64\n",
      "\n",
      "Распределение price в тестовой выборке после oversampling:\n",
      "price\n",
      "3742     33\n",
      "559      33\n",
      "8403     33\n",
      "1238     33\n",
      "1243     33\n",
      "         ..\n",
      "1149     33\n",
      "2401     33\n",
      "958      33\n",
      "702      33\n",
      "14618    33\n",
      "Name: count, Length: 5338, dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Разделение на обучающую и тестовую выборки\n",
    "train_df, test_df = train_test_split(df_cleaned, test_size=0.2, random_state=42)\n",
    "\n",
    "# Разделение обучающей выборки на обучающую и контрольную\n",
    "train_df, val_df = train_test_split(train_df, test_size=0.25, random_state=42)\n",
    "\n",
    "print(\"Размер обучающей выборки:\", len(train_df))\n",
    "print(\"Размер контрольной выборки:\", len(val_df))\n",
    "print(\"Размер тестовой выборки:\", len(test_df))\n",
    "\n",
    "def check_balance(df, name):\n",
    "    counts = df[\"price\"].value_counts()\n",
    "    print(f\"Распределение price в {name}:\")\n",
    "    print(counts)\n",
    "    print()\n",
    "\n",
    "\n",
    "check_balance(train_df, \"обучающей выборке\")\n",
    "check_balance(val_df, \"контрольной выборке\")\n",
    "check_balance(test_df, \"тестовой выборке\")\n",
    "\n",
    "def oversample(df):\n",
    "    X = df.drop(\"price\", axis=1)\n",
    "    y = df[\"price\"]\n",
    "\n",
    "    oversampler = RandomOverSampler(random_state=42)\n",
    "    X_resampled, y_resampled = oversampler.fit_resample(X, y) # type: ignore\n",
    "\n",
    "    resampled_df = pd.concat([X_resampled, y_resampled], axis=1)\n",
    "    return resampled_df\n",
    "\n",
    "\n",
    "train_df_oversampled = oversample(train_df)\n",
    "val_df_oversampled = oversample(val_df)\n",
    "test_df_oversampled = oversample(test_df)\n",
    "\n",
    "check_balance(train_df_oversampled, \"обучающей выборке после oversampling\")\n",
    "check_balance(val_df_oversampled, \"контрольной выборке после oversampling\")\n",
    "check_balance(test_df_oversampled, \"тестовой выборке после oversampling\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Распределение price в обучающей выборке после oversampling:\n",
      "price\n",
      "5076    80\n",
      "1789    80\n",
      "3931    80\n",
      "1263    80\n",
      "2026    80\n",
      "        ..\n",
      "3678    80\n",
      "4592    80\n",
      "516     80\n",
      "7152    80\n",
      "2353    80\n",
      "Name: count, Length: 9496, dtype: int64\n",
      "\n",
      "Распределение price в контрольной выборке после oversampling:\n",
      "price\n",
      "966      34\n",
      "13638    34\n",
      "3669     34\n",
      "1052     34\n",
      "2818     34\n",
      "         ..\n",
      "4032     34\n",
      "544      34\n",
      "3362     34\n",
      "6559     34\n",
      "792      34\n",
      "Name: count, Length: 5383, dtype: int64\n",
      "\n",
      "Распределение price в тестовой выборке после oversampling:\n",
      "price\n",
      "3742     33\n",
      "559      33\n",
      "8403     33\n",
      "1238     33\n",
      "1243     33\n",
      "         ..\n",
      "1149     33\n",
      "2401     33\n",
      "958      33\n",
      "702      33\n",
      "14618    33\n",
      "Name: count, Length: 5338, dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "\n",
    "def oversample(df):\n",
    "    X = df.drop(\"price\", axis=1)\n",
    "    y = df[\"price\"]\n",
    "\n",
    "    oversampler = RandomOverSampler(random_state=42)\n",
    "    X_resampled, y_resampled = oversampler.fit_resample(X, y) # type: ignore\n",
    "\n",
    "    resampled_df = pd.concat([X_resampled, y_resampled], axis=1)\n",
    "    return resampled_df\n",
    "\n",
    "\n",
    "train_df_oversampled = oversample(train_df)\n",
    "val_df_oversampled = oversample(val_df)\n",
    "test_df_oversampled = oversample(test_df)\n",
    "\n",
    "check_balance(train_df_oversampled, \"обучающей выборке после oversampling\")\n",
    "check_balance(val_df_oversampled, \"контрольной выборке после oversampling\")\n",
    "check_balance(test_df_oversampled, \"тестовой выборке после oversampling\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "aisenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
