| \n", " | Survived | \n", "Pclass | \n", "Name | \n", "Sex | \n", "Age | \n", "SibSp | \n", "Parch | \n", "Ticket | \n", "Fare | \n", "Cabin | \n", "Embarked | \n", "AgeFillNA | \n", "AgeFillMedian | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| PassengerId | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 887 | \n", "0 | \n", "2 | \n", "Montvila, Rev. Juozas | \n", "male | \n", "27.0 | \n", "0 | \n", "0 | \n", "211536 | \n", "13.00 | \n", "NaN | \n", "S | \n", "27.0 | \n", "27.0 | \n", "
| 888 | \n", "1 | \n", "1 | \n", "Graham, Miss. Margaret Edith | \n", "female | \n", "19.0 | \n", "0 | \n", "0 | \n", "112053 | \n", "30.00 | \n", "B42 | \n", "S | \n", "19.0 | \n", "19.0 | \n", "
| 889 | \n", "0 | \n", "3 | \n", "Johnston, Miss. Catherine Helen \"Carrie\" | \n", "female | \n", "NaN | \n", "1 | \n", "2 | \n", "W./C. 6607 | \n", "23.45 | \n", "NaN | \n", "S | \n", "0.0 | \n", "28.0 | \n", "
| 890 | \n", "1 | \n", "1 | \n", "Behr, Mr. Karl Howell | \n", "male | \n", "26.0 | \n", "0 | \n", "0 | \n", "111369 | \n", "30.00 | \n", "C148 | \n", "C | \n", "26.0 | \n", "26.0 | \n", "
| 891 | \n", "0 | \n", "3 | \n", "Dooley, Mr. Patrick | \n", "male | \n", "32.0 | \n", "0 | \n", "0 | \n", "370376 | \n", "7.75 | \n", "NaN | \n", "Q | \n", "32.0 | \n", "32.0 | \n", "
| \n", " | Survived | \n", "Pclass | \n", "Name | \n", "Sex | \n", "Age | \n", "SibSp | \n", "Parch | \n", "Ticket | \n", "Fare | \n", "Cabin | \n", "Embarked | \n", "AgeFillNA | \n", "AgeFillMedian | \n", "AgeCopy | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| PassengerId | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 887 | \n", "0 | \n", "2 | \n", "Montvila, Rev. Juozas | \n", "male | \n", "27.0 | \n", "0 | \n", "0 | \n", "211536 | \n", "13.00 | \n", "NaN | \n", "S | \n", "27.0 | \n", "27.0 | \n", "27.0 | \n", "
| 888 | \n", "1 | \n", "1 | \n", "Graham, Miss. Margaret Edith | \n", "female | \n", "19.0 | \n", "0 | \n", "0 | \n", "112053 | \n", "30.00 | \n", "B42 | \n", "S | \n", "19.0 | \n", "19.0 | \n", "19.0 | \n", "
| 889 | \n", "0 | \n", "3 | \n", "Johnston, Miss. Catherine Helen \"Carrie\" | \n", "female | \n", "NaN | \n", "1 | \n", "2 | \n", "W./C. 6607 | \n", "23.45 | \n", "NaN | \n", "S | \n", "0.0 | \n", "28.0 | \n", "0.0 | \n", "
| 890 | \n", "1 | \n", "1 | \n", "Behr, Mr. Karl Howell | \n", "male | \n", "26.0 | \n", "0 | \n", "0 | \n", "111369 | \n", "30.00 | \n", "C148 | \n", "C | \n", "26.0 | \n", "26.0 | \n", "26.0 | \n", "
| 891 | \n", "0 | \n", "3 | \n", "Dooley, Mr. Patrick | \n", "male | \n", "32.0 | \n", "0 | \n", "0 | \n", "370376 | \n", "7.75 | \n", "NaN | \n", "Q | \n", "32.0 | \n", "32.0 | \n", "32.0 | \n", "
![](\"assets/lec2-split.png\")
"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"# Функция для создания выборок\n",
"from sklearn.model_selection import train_test_split\n",
"\n",
"\n",
"def split_stratified_into_train_val_test(\n",
" df_input,\n",
" stratify_colname=\"y\",\n",
" frac_train=0.6,\n",
" frac_val=0.15,\n",
" frac_test=0.25,\n",
" random_state=None,\n",
"):\n",
" \"\"\"\n",
" Splits a Pandas dataframe into three subsets (train, val, and test)\n",
" following fractional ratios provided by the user, where each subset is\n",
" stratified by the values in a specific column (that is, each subset has\n",
" the same relative frequency of the values in the column). It performs this\n",
" splitting by running train_test_split() twice.\n",
"\n",
" Parameters\n",
" ----------\n",
" df_input : Pandas dataframe\n",
" Input dataframe to be split.\n",
" stratify_colname : str\n",
" The name of the column that will be used for stratification. Usually\n",
" this column would be for the label.\n",
" frac_train : float\n",
" frac_val : float\n",
" frac_test : float\n",
" The ratios with which the dataframe will be split into train, val, and\n",
" test data. The values should be expressed as float fractions and should\n",
" sum to 1.0.\n",
" random_state : int, None, or RandomStateInstance\n",
" Value to be passed to train_test_split().\n",
"\n",
" Returns\n",
" -------\n",
" df_train, df_val, df_test :\n",
" Dataframes containing the three splits.\n",
" \"\"\"\n",
"\n",
" if frac_train + frac_val + frac_test != 1.0:\n",
" raise ValueError(\n",
" \"fractions %f, %f, %f do not add up to 1.0\"\n",
" % (frac_train, frac_val, frac_test)\n",
" )\n",
"\n",
" if stratify_colname not in df_input.columns:\n",
" raise ValueError(\"%s is not a column in the dataframe\" % (stratify_colname))\n",
"\n",
" X = df_input # Contains all columns.\n",
" y = df_input[\n",
" [stratify_colname]\n",
" ] # Dataframe of just the column on which to stratify.\n",
"\n",
" # Split original dataframe into train and temp dataframes.\n",
" df_train, df_temp, y_train, y_temp = train_test_split(\n",
" X, y, stratify=y, test_size=(1.0 - frac_train), random_state=random_state\n",
" )\n",
"\n",
" # Split the temp dataframe into val and test dataframes.\n",
" relative_frac_test = frac_test / (frac_val + frac_test)\n",
" df_val, df_test, y_val, y_test = train_test_split(\n",
" df_temp,\n",
" y_temp,\n",
" stratify=y_temp,\n",
" test_size=relative_frac_test,\n",
" random_state=random_state,\n",
" )\n",
"\n",
" assert len(df_input) == len(df_train) + len(df_val) + len(df_test)\n",
"\n",
" return df_train, df_val, df_test"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pclass\n",
"3 491\n",
"1 216\n",
"2 184\n",
"Name: count, dtype: int64\n",
"Обучающая выборка: (534, 3)\n",
"Pclass\n",
"3 294\n",
"1 130\n",
"2 110\n",
"Name: count, dtype: int64\n",
"Контрольная выборка: (178, 3)\n",
"Pclass\n",
"3 98\n",
"1 43\n",
"2 37\n",
"Name: count, dtype: int64\n",
"Тестовая выборка: (179, 3)\n",
"Pclass\n",
"3 99\n",
"1 43\n",
"2 37\n",
"Name: count, dtype: int64\n"
]
}
],
"source": [
"# Вывод распределения количества наблюдений по меткам (классам)\n",
"print(df.Pclass.value_counts())\n",
"\n",
"data = df[[\"Pclass\", \"Survived\", \"AgeFillMedian\"]].copy()\n",
"\n",
"df_train, df_val, df_test = split_stratified_into_train_val_test(\n",
" data, stratify_colname=\"Pclass\", frac_train=0.60, frac_val=0.20, frac_test=0.20\n",
")\n",
"\n",
"print(\"Обучающая выборка: \", df_train.shape)\n",
"print(df_train.Pclass.value_counts())\n",
"\n",
"print(\"Контрольная выборка: \", df_val.shape)\n",
"print(df_val.Pclass.value_counts())\n",
"\n",
"print(\"Тестовая выборка: \", df_test.shape)\n",
"print(df_test.Pclass.value_counts())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Выборка с избытком (oversampling)\n",
"\n",
"https://www.blog.trainindata.com/oversampling-techniques-for-imbalanced-data/\n",
"\n",
"https://datacrayon.com/machine-learning/class-imbalance-and-oversampling/\n",
"\n",
"Выборка с недостатком (undersampling)\n",
"\n",
"https://machinelearningmastery.com/random-oversampling-and-undersampling-for-imbalanced-classification/\n",
"\n",
"Библиотека imbalanced-learn\n",
"\n",
"https://imbalanced-learn.org/stable/"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Обучающая выборка: (534, 3)\n",
"Pclass\n",
"3 294\n",
"1 130\n",
"2 110\n",
"Name: count, dtype: int64\n",
"Обучающая выборка после oversampling: (864, 3)\n",
"Pclass\n",
"3 294\n",
"2 290\n",
"1 280\n",
"Name: count, dtype: int64\n"
]
},
{
"data": {
"text/html": [
"| \n", " | Pclass | \n", "Survived | \n", "AgeFillMedian | \n", "
|---|---|---|---|
| 0 | \n", "3 | \n", "0 | \n", "28.000000 | \n", "
| 1 | \n", "3 | \n", "0 | \n", "32.000000 | \n", "
| 2 | \n", "3 | \n", "1 | \n", "28.000000 | \n", "
| 3 | \n", "1 | \n", "0 | \n", "45.000000 | \n", "
| 4 | \n", "3 | \n", "0 | \n", "7.000000 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "
| 859 | \n", "2 | \n", "0 | \n", "26.887761 | \n", "
| 860 | \n", "2 | \n", "1 | \n", "0.890459 | \n", "
| 861 | \n", "2 | \n", "0 | \n", "17.481437 | \n", "
| 862 | \n", "2 | \n", "0 | \n", "17.078473 | \n", "
| 863 | \n", "2 | \n", "1 | \n", "17.220445 | \n", "
864 rows × 3 columns
\n", "