| \n", + " | Survived | \n", + "Pclass | \n", + "Name | \n", + "Sex | \n", + "Age | \n", + "SibSp | \n", + "Parch | \n", + "Ticket | \n", + "Fare | \n", + "Cabin | \n", + "Embarked | \n", + "AgeFillNA | \n", + "AgeFillMedian | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| PassengerId | \n", + "\n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " |
| 887 | \n", + "0 | \n", + "2 | \n", + "Montvila, Rev. Juozas | \n", + "male | \n", + "27.0 | \n", + "0 | \n", + "0 | \n", + "211536 | \n", + "13.00 | \n", + "NaN | \n", + "S | \n", + "27.0 | \n", + "27.0 | \n", + "
| 888 | \n", + "1 | \n", + "1 | \n", + "Graham, Miss. Margaret Edith | \n", + "female | \n", + "19.0 | \n", + "0 | \n", + "0 | \n", + "112053 | \n", + "30.00 | \n", + "B42 | \n", + "S | \n", + "19.0 | \n", + "19.0 | \n", + "
| 889 | \n", + "0 | \n", + "3 | \n", + "Johnston, Miss. Catherine Helen \"Carrie\" | \n", + "female | \n", + "NaN | \n", + "1 | \n", + "2 | \n", + "W./C. 6607 | \n", + "23.45 | \n", + "NaN | \n", + "S | \n", + "0.0 | \n", + "28.0 | \n", + "
| 890 | \n", + "1 | \n", + "1 | \n", + "Behr, Mr. Karl Howell | \n", + "male | \n", + "26.0 | \n", + "0 | \n", + "0 | \n", + "111369 | \n", + "30.00 | \n", + "C148 | \n", + "C | \n", + "26.0 | \n", + "26.0 | \n", + "
| 891 | \n", + "0 | \n", + "3 | \n", + "Dooley, Mr. Patrick | \n", + "male | \n", + "32.0 | \n", + "0 | \n", + "0 | \n", + "370376 | \n", + "7.75 | \n", + "NaN | \n", + "Q | \n", + "32.0 | \n", + "32.0 | \n", + "
| \n", + " | Survived | \n", + "Pclass | \n", + "Name | \n", + "Sex | \n", + "Age | \n", + "SibSp | \n", + "Parch | \n", + "Ticket | \n", + "Fare | \n", + "Cabin | \n", + "Embarked | \n", + "AgeFillNA | \n", + "AgeFillMedian | \n", + "AgeCopy | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| PassengerId | \n", + "\n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " | \n", + " |
| 887 | \n", + "0 | \n", + "2 | \n", + "Montvila, Rev. Juozas | \n", + "male | \n", + "27.0 | \n", + "0 | \n", + "0 | \n", + "211536 | \n", + "13.00 | \n", + "NaN | \n", + "S | \n", + "27.0 | \n", + "27.0 | \n", + "27.0 | \n", + "
| 888 | \n", + "1 | \n", + "1 | \n", + "Graham, Miss. Margaret Edith | \n", + "female | \n", + "19.0 | \n", + "0 | \n", + "0 | \n", + "112053 | \n", + "30.00 | \n", + "B42 | \n", + "S | \n", + "19.0 | \n", + "19.0 | \n", + "19.0 | \n", + "
| 889 | \n", + "0 | \n", + "3 | \n", + "Johnston, Miss. Catherine Helen \"Carrie\" | \n", + "female | \n", + "NaN | \n", + "1 | \n", + "2 | \n", + "W./C. 6607 | \n", + "23.45 | \n", + "NaN | \n", + "S | \n", + "0.0 | \n", + "28.0 | \n", + "0.0 | \n", + "
| 890 | \n", + "1 | \n", + "1 | \n", + "Behr, Mr. Karl Howell | \n", + "male | \n", + "26.0 | \n", + "0 | \n", + "0 | \n", + "111369 | \n", + "30.00 | \n", + "C148 | \n", + "C | \n", + "26.0 | \n", + "26.0 | \n", + "26.0 | \n", + "
| 891 | \n", + "0 | \n", + "3 | \n", + "Dooley, Mr. Patrick | \n", + "male | \n", + "32.0 | \n", + "0 | \n", + "0 | \n", + "370376 | \n", + "7.75 | \n", + "NaN | \n", + "Q | \n", + "32.0 | \n", + "32.0 | \n", + "32.0 | \n", + "
![](\"assets/lec2-split.png\")
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Функция для создания выборок\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "\n",
+ "def split_stratified_into_train_val_test(\n",
+ " df_input,\n",
+ " stratify_colname=\"y\",\n",
+ " frac_train=0.6,\n",
+ " frac_val=0.15,\n",
+ " frac_test=0.25,\n",
+ " random_state=None,\n",
+ "):\n",
+ " \"\"\"\n",
+ " Splits a Pandas dataframe into three subsets (train, val, and test)\n",
+ " following fractional ratios provided by the user, where each subset is\n",
+ " stratified by the values in a specific column (that is, each subset has\n",
+ " the same relative frequency of the values in the column). It performs this\n",
+ " splitting by running train_test_split() twice.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " df_input : Pandas dataframe\n",
+ " Input dataframe to be split.\n",
+ " stratify_colname : str\n",
+ " The name of the column that will be used for stratification. Usually\n",
+ " this column would be for the label.\n",
+ " frac_train : float\n",
+ " frac_val : float\n",
+ " frac_test : float\n",
+ " The ratios with which the dataframe will be split into train, val, and\n",
+ " test data. The values should be expressed as float fractions and should\n",
+ " sum to 1.0.\n",
+ " random_state : int, None, or RandomStateInstance\n",
+ " Value to be passed to train_test_split().\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " df_train, df_val, df_test :\n",
+ " Dataframes containing the three splits.\n",
+ " \"\"\"\n",
+ "\n",
+ " if frac_train + frac_val + frac_test != 1.0:\n",
+ " raise ValueError(\n",
+ " \"fractions %f, %f, %f do not add up to 1.0\"\n",
+ " % (frac_train, frac_val, frac_test)\n",
+ " )\n",
+ "\n",
+ " if stratify_colname not in df_input.columns:\n",
+ " raise ValueError(\"%s is not a column in the dataframe\" % (stratify_colname))\n",
+ "\n",
+ " X = df_input # Contains all columns.\n",
+ " y = df_input[\n",
+ " [stratify_colname]\n",
+ " ] # Dataframe of just the column on which to stratify.\n",
+ "\n",
+ " # Split original dataframe into train and temp dataframes.\n",
+ " df_train, df_temp, y_train, y_temp = train_test_split(\n",
+ " X, y, stratify=y, test_size=(1.0 - frac_train), random_state=random_state\n",
+ " )\n",
+ "\n",
+ " # Split the temp dataframe into val and test dataframes.\n",
+ " relative_frac_test = frac_test / (frac_val + frac_test)\n",
+ " df_val, df_test, y_val, y_test = train_test_split(\n",
+ " df_temp,\n",
+ " y_temp,\n",
+ " stratify=y_temp,\n",
+ " test_size=relative_frac_test,\n",
+ " random_state=random_state,\n",
+ " )\n",
+ "\n",
+ " assert len(df_input) == len(df_train) + len(df_val) + len(df_test)\n",
+ "\n",
+ " return df_train, df_val, df_test"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Pclass\n",
+ "3 491\n",
+ "1 216\n",
+ "2 184\n",
+ "Name: count, dtype: int64\n",
+ "Обучающая выборка: (534, 3)\n",
+ "Pclass\n",
+ "3 294\n",
+ "1 130\n",
+ "2 110\n",
+ "Name: count, dtype: int64\n",
+ "Контрольная выборка: (178, 3)\n",
+ "Pclass\n",
+ "3 98\n",
+ "1 43\n",
+ "2 37\n",
+ "Name: count, dtype: int64\n",
+ "Тестовая выборка: (179, 3)\n",
+ "Pclass\n",
+ "3 99\n",
+ "1 43\n",
+ "2 37\n",
+ "Name: count, dtype: int64\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Вывод распределения количества наблюдений по меткам (классам)\n",
+ "print(df.Pclass.value_counts())\n",
+ "\n",
+ "data = df[[\"Pclass\", \"Survived\", \"AgeFillMedian\"]].copy()\n",
+ "\n",
+ "df_train, df_val, df_test = split_stratified_into_train_val_test(\n",
+ " data, stratify_colname=\"Pclass\", frac_train=0.60, frac_val=0.20, frac_test=0.20\n",
+ ")\n",
+ "\n",
+ "print(\"Обучающая выборка: \", df_train.shape)\n",
+ "print(df_train.Pclass.value_counts())\n",
+ "\n",
+ "print(\"Контрольная выборка: \", df_val.shape)\n",
+ "print(df_val.Pclass.value_counts())\n",
+ "\n",
+ "print(\"Тестовая выборка: \", df_test.shape)\n",
+ "print(df_test.Pclass.value_counts())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Выборка с избытком (oversampling)\n",
+ "\n",
+ "https://www.blog.trainindata.com/oversampling-techniques-for-imbalanced-data/\n",
+ "\n",
+ "https://datacrayon.com/machine-learning/class-imbalance-and-oversampling/\n",
+ "\n",
+ "Выборка с недостатком (undersampling)\n",
+ "\n",
+ "https://machinelearningmastery.com/random-oversampling-and-undersampling-for-imbalanced-classification/\n",
+ "\n",
+ "Библиотека imbalanced-learn\n",
+ "\n",
+ "https://imbalanced-learn.org/stable/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Обучающая выборка: (534, 3)\n",
+ "Pclass\n",
+ "3 294\n",
+ "1 130\n",
+ "2 110\n",
+ "Name: count, dtype: int64\n",
+ "Обучающая выборка после oversampling: (864, 3)\n",
+ "Pclass\n",
+ "3 294\n",
+ "2 290\n",
+ "1 280\n",
+ "Name: count, dtype: int64\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "| \n", + " | Pclass | \n", + "Survived | \n", + "AgeFillMedian | \n", + "
|---|---|---|---|
| 0 | \n", + "3 | \n", + "0 | \n", + "28.000000 | \n", + "
| 1 | \n", + "3 | \n", + "0 | \n", + "32.000000 | \n", + "
| 2 | \n", + "3 | \n", + "1 | \n", + "28.000000 | \n", + "
| 3 | \n", + "1 | \n", + "0 | \n", + "45.000000 | \n", + "
| 4 | \n", + "3 | \n", + "0 | \n", + "7.000000 | \n", + "
| ... | \n", + "... | \n", + "... | \n", + "... | \n", + "
| 859 | \n", + "2 | \n", + "0 | \n", + "26.887761 | \n", + "
| 860 | \n", + "2 | \n", + "1 | \n", + "0.890459 | \n", + "
| 861 | \n", + "2 | \n", + "0 | \n", + "17.481437 | \n", + "
| 862 | \n", + "2 | \n", + "0 | \n", + "17.078473 | \n", + "
| 863 | \n", + "2 | \n", + "1 | \n", + "17.220445 | \n", + "
864 rows × 3 columns
\n", + "