{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from IPython.core.interactiveshell import InteractiveShell\n", "import math\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt # for plotting \n", "import seaborn as sns # for plotting\n", "\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.impute import SimpleImputer, MissingIndicator\n", "from sklearn.experimental import enable_iterative_imputer\n", "from sklearn.impute import IterativeImputer\n", "from sklearn.pipeline import FeatureUnion, make_pipeline" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# set up some notebook display defaults\n", "InteractiveShell.ast_node_interactivity = \"all\"\n", "%matplotlib inline\n", "plt.style.use('default')\n", "sns.set()\n", "pd.options.display.float_format = '{:,.2f}'.format" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# paths to datasets\n", "kaggle_trn = \"./data/titanic/train.csv\"\n", "kaggle_tst = \"./data/titanic/test.csv\"\n", "oma_trn_3 = \"./data/titanic/oma_trn_3.csv\"\n", "oma_tst_3 = \"./data/titanic/oma_tst_3.csv\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# load the datasets currently of interest\n", "k_trn = pd.read_csv(oma_trn_3)\n", "k_tst = pd.read_csv(oma_tst_3)\n", "k_all = k_trn\n", "k_all = pd.concat([k_all, k_tst], ignore_index=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "y_trn = k_trn['Survived']\n", "\n", "# start fresh\n", "features = ['PassengerId', 'Pclass', 'Sex', 'SibSp', 'Parch', 'Age', 'Title']\n", "X_trn = k_trn[features].copy()\n", "X_tst = k_tst[features].copy()\n", "# let's load the Kaggle datasets so we can get the original Age data\n", "kg_trn = pd.read_csv(kaggle_trn)\n", "kg_tst = pd.read_csv(kaggle_tst)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# now let's replace the Age data in my versions of the datasets with that from the Kaggle datasets\n", "# for test will add kaggle column as well\n", "X_trn.rename(columns={\"Age\": \"iiAge\"}, inplace=True)\n", "X_trn[\"Age\"] = kg_trn[\"Age\"]\n", "X_tst.rename(columns={\"Age\": \"iiAge\"}, inplace=True)\n", "X_tst[\"Age\"] = kg_tst[\"Age\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X_trn.head()\n", "X_trn.tail()\n", "X_trn[\"Age\"].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "mean_trn = k_trn[k_trn[\"Title\"] == \"Master\"][\"Age\"].mean()\n", "mean_tst = k_tst[k_tst[\"Title\"] == \"Master\"][\"Age\"].mean()\n", "cnt_trn = k_trn[k_trn[\"Title\"] == \"Master\"][\"Title\"].count()\n", "cnt_tst = k_tst[k_tst[\"Title\"] == \"Master\"][\"Title\"].count()\n", "mean_all = ((mean_trn * cnt_trn) + (mean_tst * cnt_tst)) / (cnt_trn + cnt_tst)\n", "print(f\"Mean Age for 'Master': training set {mean_trn:.3f} ({cnt_trn}), test set {mean_tst:.3f} ({cnt_tst}) -> {mean_all:.3f}\")\n", "\n", "X_trn[(X_trn[\"Title\"] == \"Master\") & (X_trn[\"Age\"].isnull())].head()\n", "X_tst[(X_tst[\"Title\"] == \"Master\") & (X_tst[\"Age\"].isnull())].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X_trn.loc[(X_trn[\"Title\"] == \"Master\") & (X_trn[\"Age\"].isnull()), \"Age\"] = mean_all\n", "X_tst.loc[(X_tst[\"Title\"] == \"Master\") & (X_tst[\"Age\"].isnull()), \"Age\"] = mean_all" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# let's check\n", "pid_trn = [66, 160, 177, 710]\n", "pid_tst = [1136, 1231, 1236, 1309]\n", "X_trn[X_trn[\"PassengerId\"].isin(pid_trn)]\n", "X_tst[X_tst[\"PassengerId\"].isin(pid_tst)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# okay, not on to the IterativeImputer\n", "min_age = min(X_trn[\"Age\"].min(), X_tst[\"Age\"].min())\n", "max_age = max(X_trn[\"Age\"].max(), X_tst[\"Age\"].max())\n", "print(min_age, max_age)\n", "transformer = FeatureUnion(\n", " transformer_list=[\n", " ('features', IterativeImputer(max_iter=10, min_value=min_age, max_value=max_age, random_state=0)),\n", " ('indicators', MissingIndicator())])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "features = [\"PassengerId\", \"Pclass\", \"Sex\", \"SibSp\", \"Parch\",\t\"Title\", \"Age\"]\n", "X_trn = pd.get_dummies(X_trn[features])\n", "X_tst = pd.get_dummies(X_tst[features])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X_trn.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# let's train, and transform, our imputer on X_trn, and have look\n", "trn_cols = X_trn.columns.tolist()\n", "trn_cols.append(\"AgeMissing\")\n", "X_trn_trans = transformer.fit_transform(X_trn, y_trn)\n", "X_trn_trans = pd.DataFrame(X_trn_trans, columns=trn_cols)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "disp_cols = [\"PassengerId\", \"Pclass\", \"Sex_female\", \"Sex_male\", \"SibSp\", \"Parch\", \"Age\"]\n", "# X_trn_trans[disp_cols].tail()\n", "X_trn_trans[disp_cols].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# looks better, do the same for X_tst\n", "tst_cols = X_tst.columns.tolist()\n", "tst_cols.append(\"AgeMissing\")\n", "X_tst_trans = transformer.transform(X_tst)\n", "X_tst_trans = pd.DataFrame(X_tst_trans, columns=tst_cols)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "disp_cols = [\"PassengerId\", \"Pclass\", \"Sex_female\", \"Sex_male\", \"SibSp\", \"Parch\", \"Age\"]\n", "# X_trn_trans[disp_cols].tail()\n", "X_tst_trans[disp_cols].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# new updated training dataset dataframe\n", "k_trn_2 = k_trn.copy()\n", "k_trn_2 = k_trn_2.drop(\"AgeMissing\", axis=1)\n", "k_trn_2[:].Age = X_trn_trans[:].Age\n", "# k_trn_2[:].AgeMissing = X_trn_trans[:].AgeMissing\n", "k_trn_2 = pd.concat([k_trn_2, X_trn_trans[:].AgeMissing], axis=1)\n", "# new updated testing dataset dataframe\n", "k_tst_2 = k_tst.copy()\n", "k_tst_2 = k_tst_2.drop(\"AgeMissing\", axis=1)\n", "k_tst_2[:].Age = X_tst_trans[:].Age\n", "# k_tst_2[:].AgeMissing = X_tst_trans[:].AgeMissing\n", "k_tst_2 = pd.concat([k_tst_2, X_tst_trans[:].AgeMissing], axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "k_trn_2.describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "k_tst_2.describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "k_trn_2.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# glad I did that, almost forgot to update AgeBin again\n", "bin_thresholds = [0, 15, 30, 40, 59, 90]\n", "bin_labels = ['0-15', '16-29', '30-40', '41-59', '60+']\n", "k_trn_2['AgeBin'] = pd.cut(k_trn['Age'], bins=bin_thresholds, labels=bin_labels)\n", "k_tst_2['AgeBin'] = pd.cut(k_tst['Age'], bins=bin_thresholds, labels=bin_labels)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "k_trn_2.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "k_tst_2.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# save updated datasets to our CSV files\n", "k_trn_2.to_csv(oma_trn_3, index=False)\n", "k_tst_2.to_csv(oma_tst_3, index=False)" ] } ], "metadata": { "interpreter": { "hash": "a27d3f2bf68df5402465348834a2195030d3fc5bfc8e594e2a17c8c7e2447c85" }, "kernelspec": { "display_name": "Python 3.9.2 64-bit ('ds-3.9': conda)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.2" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }