201 KiB
201 KiB
Загрузка набора данных¶
In [732]:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import math
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from imblearn.over_sampling import RandomOverSampler
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import PolynomialFeatures
from sklearn import metrics
from sklearn import linear_model, tree, neighbors, ensemble
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import FunctionTransformer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn import set_config
from sklearn.pipeline import make_pipeline
df = pd.read_csv("../data/car_price_prediction.csv")
df = df.drop(columns=["ID"])
random_state = 9
set_config(transform_output="pandas")
df
Out[732]:
Анализ датасета и очистка данных¶
In [733]:
df.dtypes
Out[733]:
In [734]:
df["Engine volume"] = df["Engine volume"].str.replace("Turbo", "")
df["Engine volume"] = pd.to_numeric(df["Engine volume"])
df["Engine volume"].unique()
Out[734]:
In [735]:
df["Mileage"] = df["Mileage"].str.replace("km", "")
df["Mileage"] = df["Mileage"].astype("int64")
df["Mileage"].unique()
Out[735]:
In [736]:
df["Levy"] = df["Levy"].replace("-", "0")
df["Levy"] = df["Levy"].astype("int64")
df["Levy"].unique()
Out[736]:
In [737]:
df["Cylinders"] = df["Cylinders"].astype("int64")
df["Cylinders"].unique()
Out[737]:
In [738]:
df["Doors"].unique()
Out[738]:
In [739]:
df["Doors"] = df["Doors"].map(
{"02-Mar": "Двухдверный", "04-May": "Четырехдверный", ">5": "Многодверный"}
)
df["Doors"].unique()
Out[739]:
In [740]:
sorted_df = df.sort_values(by="Price")
sorted_df["Price"].unique()
Out[740]:
In [741]:
print(f"Количество строк до удаления некорректных значений: {len(df)}")
df = df[df["Price"] >= 500]
print(f"Количество строк после удаления некорректных значений: {len(df)}")
In [742]:
sorted_df = df.sort_values(by="Price")
sorted_df["Price"].unique()
Out[742]:
In [743]:
sorted_df = df.sort_values(by="Prod. year")
sorted_df["Prod. year"].unique()
Out[743]:
In [744]:
df
Out[744]:
Очистка дубликатов и пропущенных значений¶
In [745]:
df.duplicated().sum()
Out[745]:
In [746]:
df.drop_duplicates(inplace=True)
# df.duplicated().sum()
In [747]:
df.isna().sum()
Out[747]:
Очистка выбросов¶
In [748]:
df.dtypes
Out[748]:
In [749]:
numeric_features_with_outliers = [
"Price",
"Levy",
"Mileage",
"Prod. year",
]
i = 1
for col in numeric_features_with_outliers:
plt.figure(figsize=(4, 30))
plt.subplot(6, 1, i)
df.boxplot(column=col)
i += 1
In [750]:
def remove_outliers(df, column):
Q1 = df[column].quantile(0.25)
Q3 = df[column].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
return df[(df[column] >= lower_bound) & (df[column] <= upper_bound)]
print(f"Количество строк до удаления выбросов: {len(df)}")
for column in numeric_features_with_outliers:
df = remove_outliers(df, column)
print(f"Количество строк после удаления выбросов: {len(df)}")
In [751]:
i = 1
for col in numeric_features_with_outliers:
plt.figure(figsize=(4, 30))
plt.subplot(6, 1, i)
df.boxplot(column=col)
i += 1
Формирование конвейера для классификации данных¶
In [752]:
df.dtypes
Out[752]:
In [753]:
columns_to_drop = [
"Model",
"Manufacturer",
"Color",
"Doors",
"Cylinders",
"Mileage",
]
num_columns = [
column
for column in df.columns
if column not in columns_to_drop and df[column].dtype != "object"
]
cat_columns = [
column
for column in df.columns
if column not in columns_to_drop and df[column].dtype == "object"
]
cat_cols_for_one_hot_enc = [
"Leather interior",
"Category",
"Fuel type",
"Gear box type",
"Drive wheels",
# "Doors",
"Wheel",
]
cat_cols_for_num_enc = [
# "Model",
# "Manufacturer",
# "Color",
]
num_imputer = SimpleImputer(strategy="median")
num_scaler = StandardScaler()
preprocessing_num = Pipeline(
[
("imputer", num_imputer),
("scaler", num_scaler),
]
)
cat_imputer = SimpleImputer(strategy="constant", fill_value="unknown")
cat_one_hot_encoder = OneHotEncoder(
handle_unknown="ignore", sparse_output=False, drop="first"
)
preprocessing_one_hot = Pipeline(
[
("imputer", cat_imputer),
("encoder", cat_one_hot_encoder),
]
)
preprocessing_label_enc = Pipeline(
[
("imputer", cat_imputer),
("encoder", OrdinalEncoder()),
]
)
features_preprocessing = ColumnTransformer(
verbose_feature_names_out=False,
transformers=[
("prepocessing_one_hot", preprocessing_one_hot, cat_cols_for_one_hot_enc),
("prepocessing_label_enc", preprocessing_label_enc, cat_cols_for_num_enc),
("prepocessing_num", preprocessing_num, num_columns),
],
remainder="passthrough",
)
drop_columns = ColumnTransformer(
verbose_feature_names_out=False,
transformers=[
("drop_columns", "drop", columns_to_drop),
],
remainder="passthrough",
)
pipeline_end = Pipeline(
[
("features_preprocessing", features_preprocessing),
("drop_columns", drop_columns),
]
)
Демонстрация работы конвейера для предобработки данных¶
In [754]:
preprocessing_result = pipeline_end.fit_transform(df)
df = pd.DataFrame(
preprocessing_result,
columns=pipeline_end.get_feature_names_out(),
)
df
Out[754]:
Разбиение на выборки¶
In [755]:
train_df, test_df = train_test_split(
df, test_size=0.3, random_state=42
)
print("Размеры выборок:")
print(f"Обучающая выборка: {train_df.shape[0]} записей")
print(f"Тестовая выборка: {test_df.shape[0]} записей")
In [756]:
price_y_train = train_df["Price"]
price_y_test = test_df["Price"]
train_df.drop(columns=["Price"], inplace=True)
test_df.drop(columns=["Price"], inplace=True)
train_df
Out[756]:
Формирование набора моделей¶
In [757]:
models = {
"linear": {"model": linear_model.LinearRegression(n_jobs=-1)},
# "linear_poly": {
# "model": make_pipeline(
# PolynomialFeatures(degree=2),
# linear_model.LinearRegression(fit_intercept=False, n_jobs=-1),
# )
# },
# "linear_interact": {
# "model": make_pipeline(
# PolynomialFeatures(interaction_only=True),
# linear_model.LinearRegression(fit_intercept=False, n_jobs=-1),
# )
# },
"ridge": {"model": linear_model.RidgeCV()},
"decision_tree": {
"model": tree.DecisionTreeRegressor(max_depth=7, random_state=random_state)
},
"knn": {"model": neighbors.KNeighborsRegressor(n_neighbors=7, n_jobs=-1)},
"random_forest": {
"model": ensemble.RandomForestRegressor(
max_depth=7, random_state=random_state, n_jobs=-1
)
},
}
Обучение и оценка моделей с помощью различных алгоритмов¶
In [758]:
for model_name in models.keys():
print(f"Model: {model_name}")
model = models[model_name]["model"]
fitted_model = model.fit(train_df.values, price_y_train.values.ravel())
y_train_pred = fitted_model.predict(train_df)
y_test_pred = fitted_model.predict(test_df)
models[model_name]["fitted"] = fitted_model
models[model_name]["train_preds"] = y_train_pred
models[model_name]["preds"] = y_test_pred
models[model_name]["RMSE_train"] = math.sqrt(
metrics.mean_squared_error(price_y_train, y_train_pred)
)
models[model_name]["RMSE_test"] = math.sqrt(
metrics.mean_squared_error(price_y_test, y_test_pred)
)
models[model_name]["RMAE_test"] = math.sqrt(
metrics.mean_absolute_error(price_y_test, y_test_pred)
)
models[model_name]["R2_test"] = metrics.r2_score(price_y_test, y_test_pred)
Вывод результатов оценки¶
In [759]:
reg_metrics = pd.DataFrame.from_dict(models, "index")[
["RMSE_train", "RMSE_test", "RMAE_test", "R2_test"]
]
reg_metrics.sort_values(by="RMSE_test").style.background_gradient(
cmap="viridis", low=1, high=0.3, subset=["RMSE_train", "RMSE_test"]
).background_gradient(cmap="plasma", low=0.3, high=1, subset=["RMAE_test", "R2_test"])
Out[759]: