PIbd-33_Dyakonov_R_R_MAI/lab2.1.ipynb

In [ ]:

import numpy as np
import pandas as pd
from sklearn.feature_selection import mutual_info_regression
from sklearn.model_selection import train_test_split
from imblearn.over_sampling import ADASYN
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
import seaborn as sns

# Загрузка датасета с мобильными телефонами
df = pd.read_csv("data/mobile_prices.csv")
print(df.columns)

# Кодируем все строковые столбцы в числовые
label_encoders = {}
for col in df.select_dtypes(include=["object"]).columns:
    le = LabelEncoder()
    df[col] = le.fit_transform(df[col])
    label_encoders[col] = le

# Проверка на пропуски и "зашумленные" столбцы
noisy_features = []
for col in df.columns:
    if df[col].isnull().sum() / len(df) > 0.1:  # Если более 10% пропусков
        noisy_features.append(col)
print(f"Зашумленные столбцы: {noisy_features}")

# Проверка на смещение
skewness = df.skew()
print(f"Смещение: {skewness}")

skewed_features = skewness[abs(skewness) > 1].index.tolist()
print(f"Сильно смещенные столбцы: {skewed_features}")

# Поиск выбросов
for col in df.select_dtypes(include=["number"]).columns:
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    outliers = df[col][(df[col] < lower_bound) | (df[col] > upper_bound)]
    print(f"Выбросы в столбце '{col}':\n{outliers}\n")

# Визуализация выбросов
numeric_cols = df.select_dtypes(include=["number"]).columns

plt.figure(figsize=(12, 8))
for i, col in enumerate(numeric_cols, 1):
    plt.subplot(len(numeric_cols) // 3 + 1, 3, i)
    sns.boxplot(data=df, x=col)
    plt.title(f"Boxplot for {col}")
plt.tight_layout()
plt.show()

# Логарифмирование признака 'Battery'
df["log_Battery"] = np.log(df["Battery"] + 1)

# Заполнение пропусков
df["Battery"] = df["Battery"].fillna(df["Battery"].mean())


# Функция для разбиения на train/val/test
def split_stratified_into_train_val_test(
    df_input,
    stratify_colname="y",
    frac_train=0.6,
    frac_val=0.15,
    frac_test=0.25,
    random_state=None,
):
    if frac_train + frac_val + frac_test != 1.0:
        raise ValueError(
            "fractions %f, %f, %f do not add up to 1.0"
            % (frac_train, frac_val, frac_test)
        )

    if stratify_colname not in df_input.columns:
        raise ValueError("%s is not a column in the dataframe" % (stratify_colname))

    X = df_input
    y = df_input[[stratify_colname]]

    df_train, df_temp, y_train, y_temp = train_test_split(
        X, y, stratify=y, test_size=(1.0 - frac_train), random_state=random_state
    )
    relative_frac_test = frac_test / (frac_val + frac_test)
    df_val, df_test, y_val, y_test = train_test_split(
        df_temp,
        y_temp,
        stratify=y_temp,
        test_size=relative_frac_test,
        random_state=random_state,
    )

    assert len(df_input) == len(df_train) + len(df_val) + len(df_test)

    return df_train, df_val, df_test


# Разбиение на train/val/test
data = df[["Ram", "Price", "company"]].copy()

print("@data", data)

data = data.groupby("company").filter(
    lambda x: len(x) > 4
)  # убираем классы с одним элементом

df_train, df_val, df_test = split_stratified_into_train_val_test(
    data, stratify_colname="company", frac_train=0.60, frac_val=0.20, frac_test=0.20
)

print("Обучающая выборка: ", df_train.shape)
print(df_train["Ram"].value_counts())

print("Контрольная выборка: ", df_val.shape)
print(df_val["Ram"].value_counts())

print("Тестовая выборка: ", df_test.shape)
print(df_test["Ram"].value_counts())

# # Применение ADASYN для oversampling
# ada = ADASYN(n_neighbors=2)
# X_resampled, y_resampled = ada.fit_resample(df_train, df_train["company"])
# df_train_adasyn = pd.DataFrame(X_resampled)

# print("Обучающая выборка после oversampling: ", df_train_adasyn.shape)
# print(df_train_adasyn["Ram"].value_counts())