diff --git a/lab-hypothesis-testing.ipynb b/lab-hypothesis-testing.ipynb index 0cc26d5..9e3433d 100644 --- a/lab-hypothesis-testing.ipynb +++ b/lab-hypothesis-testing.ipynb @@ -51,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -278,7 +278,7 @@ "[800 rows x 11 columns]" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -297,11 +297,26 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "๐Ÿ” POKEMON DATA EXPLORATION: DATA TYPES AND DIMENSIONS\n", + "๐Ÿ“‹ Rows: 800 | Columns: 11\n" + ] + } + ], "source": [ - "#code here" + "\n", + "print(\"๐Ÿ” POKEMON DATA EXPLORATION: DATA TYPES AND DIMENSIONS\")\n", + "\n", + "# 1. Show dataset shape summary\n", + "print(f\"๐Ÿ“‹ Rows: {df.shape[0]} | Columns: {df.shape[1]}\")\n", + "\n", + "\n" ] }, { @@ -313,11 +328,241 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "#code here" + "df = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/pokemon.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "๐Ÿ“‹ Column Data Types:\n", + "Name object\n", + "Type 1 object\n", + "Type 2 object\n", + "HP int64\n", + "Attack int64\n", + "Defense int64\n", + "Sp. Atk int64\n", + "Sp. Def int64\n", + "Speed int64\n", + "Generation int64\n", + "Legendary bool\n", + "dtype: object\n" + ] + } + ], + "source": [ + "# 2. View column names and quick info\n", + "print(\"\\n๐Ÿ“‹ Column Data Types:\")\n", + "print(df.dtypes)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "๐Ÿ“‹ Column Data Types:\n", + "Name object\n", + "Type 1 object\n", + "Type 2 object\n", + "HP int64\n", + "Attack int64\n", + "Defense int64\n", + "Sp. Atk int64\n", + "Sp. Def int64\n", + "Speed int64\n", + "Generation int64\n", + "Legendary bool\n", + "dtype: object\n" + ] + } + ], + "source": [ + "# 2. View column names and quick info\n", + "print(\"\\n๐Ÿ“‹ Column Data Types:\")\n", + "print(df.dtypes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hypotheses:\\\n", + "\n", + "(H_{0}\\): \\(\\mu_{\\text{Dragon HP}} \\le \\mu_{\\text{Grass HP}}\\) (Dragon Pokรฉmon do not have more HP than Grass Pokรฉmon)\n", + "\n", + "\n", + "\n", + "\\(H_{1}\\): \\(\\mu_{\\text{Dragon HP}} > \\mu_{\\text{Grass HP}}\\) (Dragon Pokรฉmon have more HP than Grass Pokรฉmon)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "๐Ÿงช HYPOTHESIS TEST 1: DRAGON VS GRASS HP\n" + ] + } + ], + "source": [ + "print(\"๐Ÿงช HYPOTHESIS TEST 1: DRAGON VS GRASS HP\")\n", + "\n", + "# Isolate the HP for Dragon and Grass types (checking primary Type 1)\n", + "dragon_hp = df[df['Type 1'] == 'Dragon']['HP']\n", + "grass_hp = df[df['Type 1'] == 'Grass']['HP']\n", + "\n", + "# Perform Welch's T-test (equal_var=False handles unequal group sizes/variances)\n", + "t_stat, p_val_two_tailed = st.ttest_ind(dragon_hp, grass_hp, equal_var=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert two-tailed p-value to one-tailed (since we posit Dragon > Grass)\n", + "p_val_one_tailed = p_val_two_tailed / 2 if t_stat > 0 else 1 - (p_val_two_tailed)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean HP - Dragon: 83.31 | Grass: 67.27\n", + "t-statistic: 3.3350\n", + "One-tailed p-value: 0.0008\n" + ] + } + ], + "source": [ + "print(f\"Mean HP - Dragon: {dragon_hp.mean():.2f} | Grass: {grass_hp.mean():.2f}\")\n", + "print(f\"t-statistic: {t_stat:.4f}\")\n", + "print(f\"One-tailed p-value: {p_val_one_tailed:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "๐Ÿ“ข Finding: Reject H0. Dragon Pokemons have significantly more HP than Grass Pokemons on average.\n" + ] + } + ], + "source": [ + "# Check significance (5%)\n", + "alpha = 0.05\n", + "if p_val_one_tailed < alpha:\n", + " print(\"๐Ÿ“ข Finding: Reject H0. Dragon Pokemons have significantly more HP than Grass Pokemons on average.\")\n", + "else:\n", + " print(\"๐Ÿ“ข Finding: Fail to reject H0. No significant difference to prove Dragons have higher average HP.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "๐Ÿงช HYPOTHESIS TEST 2: LEGENDARY VS NON-LEGENDARY\n", + "\n", + "๐Ÿ“Š Stat: HP\n", + " t-statistic: 8.9814 | p-value: 1.0027e-13\n", + " ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different HP stats.\n", + "\n", + "๐Ÿ“Š Stat: Attack\n", + " t-statistic: 10.4381 | p-value: 2.5204e-16\n", + " ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different Attack stats.\n", + "\n", + "๐Ÿ“Š Stat: Defense\n", + " t-statistic: 7.6371 | p-value: 4.8270e-11\n", + " ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different Defense stats.\n", + "\n", + "๐Ÿ“Š Stat: Sp. Atk\n", + " t-statistic: 13.4174 | p-value: 1.5515e-21\n", + " ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different Sp. Atk stats.\n", + "\n", + "๐Ÿ“Š Stat: Sp. Def\n", + " t-statistic: 10.0157 | p-value: 2.2949e-15\n", + " ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different Sp. Def stats.\n", + "\n", + "๐Ÿ“Š Stat: Speed\n", + " t-statistic: 11.4750 | p-value: 1.0490e-18\n", + " ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different Speed stats.\n" + ] + } + ], + "source": [ + "# --- PART 2: LEGENDARY VS NON-LEGENDARY ---\n", + "print(\"\\n๐Ÿงช HYPOTHESIS TEST 2: LEGENDARY VS NON-LEGENDARY\")\n", + "\n", + "# Separate groups\n", + "legendary = df[df['Legendary'] == True]\n", + "non_legendary = df[df['Legendary'] == False]\n", + "\n", + "# Clean column mapping to fix matching issues (e.g. Sp.Atk vs Sp. Atk)\n", + "stats_to_test = ['HP', 'Attack', 'Defense', 'Sp. Atk', 'Sp. Def', 'Speed']\n", + "\n", + "# Loop through each metric and test\n", + "alpha = 0.05\n", + "for stat in stats_to_test:\n", + " # Ensure correct column string indexing\n", + " col_name = stat if stat in df.columns else (stat.replace('.', '. ') if stat.replace('.', '. ') in df.columns else None)\n", + " \n", + " if col_name:\n", + " t_stat, p_val = st.ttest_ind(legendary[col_name], non_legendary[col_name], equal_var=False)\n", + " \n", + " print(f\"\\n๐Ÿ“Š Stat: {stat}\")\n", + " print(f\" t-statistic: {t_stat:.4f} | p-value: {p_val:.4e}\")\n", + " \n", + " if p_val < alpha:\n", + " print(f\" ๐Ÿ“ข Finding: Reject H0. Legendary Pokemons have significantly different {stat} stats.\")\n", + " else:\n", + " print(f\" ๐Ÿ“ข Finding: Fail to reject H0. No significant difference found for {stat}.\")\n", + " else:\n", + " print(f\"โš ๏ธ Column {stat} not found in the dataset.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Key Insights for your Lab CommentsDragon vs. Grass: The true dataset shows that Dragon types exhibit a noticeably higher baseline base health pool (\\(p < 0.05\\)). This successfully rejects \\(H_{0}\\).Legendary vs. Non-Legendary: Legendary Pokรฉmon yield extremely small p-values (typically \\(p < 0.001\\)) across all 6 base stats. You can confidently comment that Legendary Pokรฉmon are structurally scaled differently in game balance across every single attribute." ] }, { @@ -463,6 +708,102 @@ "df.head()" ] }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "๐Ÿ” DATASET COMPONENT SUMMARY:\n", + " Houses Close to Facility (n): 6829\n", + " Houses Far from Facility (n): 10171\n", + " Mean Price (Close): $246,951.98\n", + " Mean Price (Far): $180,678.44\n", + "\n", + "๐Ÿงช HYPOTHESIS TEST METRICS:\n", + " Calculated t-statistic: 37.9923\n", + " One-tailed p-value: 1.5032e-301\n", + "\n", + "๐Ÿ“ข Finding: Reject H0. Houses close to a school or hospital are significantly more expensive.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import scipy.stats as st\n", + "\n", + "# 1. Load dataset\n", + "url = \"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/california_housing.csv\"\n", + "df = pd.read_csv(url)\n", + "\n", + "# Defined Reference Targets\n", + "school_coords = (-118, 34)\n", + "hospital_coords = (-122, 37)\n", + "\n", + "\n", + "# 2. Distance function (Optimized vector application)\n", + "def calc_euclidean(row, target):\n", + " target_lon, target_lat = target\n", + " return np.sqrt(\n", + " (row[\"longitude\"] - target_lon) ** 2\n", + " + (row[\"latitude\"] - target_lat) ** 2\n", + " )\n", + "\n", + "\n", + "# Apply calculations to append distance metrics\n", + "df[\"dist_to_school\"] = df.apply(\n", + " calc_euclidean, axis=1, target=school_coords\n", + ")\n", + "df[\"dist_to_hospital\"] = df.apply(\n", + " calc_euclidean, axis=1, target=hospital_coords\n", + ")\n", + "\n", + "# 3. Categorize locations based on the 0.50 threshold criteria\n", + "df[\"is_close\"] = (df[\"dist_to_school\"] < 0.50) | (\n", + " df[\"dist_to_hospital\"] < 0.50\n", + ")\n", + "\n", + "# Isolate target price vectors\n", + "close_prices = df[df[\"is_close\"] == True][\"median_house_value\"].dropna()\n", + "far_prices = df[df[\"is_close\"] == False][\"median_house_value\"].dropna()\n", + "\n", + "# 4. Statistical Testing: One-Tailed Welch's T-Test\n", + "# equal_var=False addresses unequal variance and uneven group sizes (Welch's correction)\n", + "t_stat, p_val_two_tailed = st.ttest_ind(\n", + " close_prices, far_prices, equal_var=False\n", + ")\n", + "\n", + "# Convert two-tailed test to a one-tailed directional output (Close > Far)\n", + "p_val_one_tailed = (\n", + " p_val_two_tailed / 2 if t_stat > 0 else 1 - (p_val_two_tailed / 2)\n", + ")\n", + "\n", + "# 5. Output Results Display\n", + "print(\"๐Ÿ” DATASET COMPONENT SUMMARY:\")\n", + "print(f\" Houses Close to Facility (n): {len(close_prices)}\")\n", + "print(f\" Houses Far from Facility (n): {len(far_prices)}\")\n", + "print(f\" Mean Price (Close): ${close_prices.mean():,.2f}\")\n", + "print(f\" Mean Price (Far): ${far_prices.mean():,.2f}\")\n", + "\n", + "print(\"\\n๐Ÿงช HYPOTHESIS TEST METRICS:\")\n", + "print(f\" Calculated t-statistic: {t_stat:.4f}\")\n", + "print(f\" One-tailed p-value: {p_val_one_tailed:.4e}\")\n", + "\n", + "alpha = 0.05\n", + "if p_val_one_tailed < alpha and t_stat > 0:\n", + " print(\n", + " \"\\n๐Ÿ“ข Finding: Reject H0. Houses close to a school or hospital are significantly more expensive.\"\n", + " )\n", + "else:\n", + " print(\n", + " \"\\n๐Ÿ“ข Finding: Fail to reject H0. Proximity to these coordinates does not yield higher average values.\"\n", + " )\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -498,7 +839,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "base", "language": "python", "name": "python3" }, @@ -512,7 +853,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.13.9" } }, "nbformat": 4,