diff --git a/.ipynb_checkpoints/lab-hypothesis-testing-checkpoint.ipynb b/.ipynb_checkpoints/lab-hypothesis-testing-checkpoint.ipynb new file mode 100644 index 0000000..d951be6 --- /dev/null +++ b/.ipynb_checkpoints/lab-hypothesis-testing-checkpoint.ipynb @@ -0,0 +1,1155 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Lab | Hypothesis Testing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Objective**\n", + "\n", + "Welcome to the Hypothesis Testing Lab, where we embark on an enlightening journey through the realm of statistical decision-making! In this laboratory, we delve into various scenarios, applying the powerful tools of hypothesis testing to scrutinize and interpret data.\n", + "\n", + "From testing the mean of a single sample (One Sample T-Test), to investigating differences between independent groups (Two Sample T-Test), and exploring relationships within dependent samples (Paired Sample T-Test), our exploration knows no bounds. Furthermore, we'll venture into the realm of Analysis of Variance (ANOVA), unraveling the complexities of comparing means across multiple groups.\n", + "\n", + "So, grab your statistical tools, prepare your hypotheses, and let's embark on this fascinating journey of exploration and discovery in the world of hypothesis testing!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Challenge 1**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this challenge, we will be working with pokemon data. The data can be found here:\n", + "\n", + "- https://raw.githubusercontent.com/data-bootcamp-v4/data/main/pokemon.csv" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "#libraries\n", + "import pandas as pd\n", + "import scipy.stats as st\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameType 1Type 2HPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
0BulbasaurGrassPoison4549496565451False
1IvysaurGrassPoison6062638080601False
2VenusaurGrassPoison808283100100801False
3Mega VenusaurGrassPoison80100123122120801False
4CharmanderFireNaN3952436050651False
....................................
795DiancieRockFairy50100150100150506True
796Mega DiancieRockFairy501601101601101106True
797Hoopa ConfinedPsychicGhost8011060150130706True
798Hoopa UnboundPsychicDark8016060170130806True
799VolcanionFireWater8011012013090706True
\n", + "

800 rows × 11 columns

\n", + "
" + ], + "text/plain": [ + " Name Type 1 Type 2 HP Attack Defense Sp. Atk Sp. Def \\\n", + "0 Bulbasaur Grass Poison 45 49 49 65 65 \n", + "1 Ivysaur Grass Poison 60 62 63 80 80 \n", + "2 Venusaur Grass Poison 80 82 83 100 100 \n", + "3 Mega Venusaur Grass Poison 80 100 123 122 120 \n", + "4 Charmander Fire NaN 39 52 43 60 50 \n", + ".. ... ... ... .. ... ... ... ... \n", + "795 Diancie Rock Fairy 50 100 150 100 150 \n", + "796 Mega Diancie Rock Fairy 50 160 110 160 110 \n", + "797 Hoopa Confined Psychic Ghost 80 110 60 150 130 \n", + "798 Hoopa Unbound Psychic Dark 80 160 60 170 130 \n", + "799 Volcanion Fire Water 80 110 120 130 90 \n", + "\n", + " Speed Generation Legendary \n", + "0 45 1 False \n", + "1 60 1 False \n", + "2 80 1 False \n", + "3 80 1 False \n", + "4 65 1 False \n", + ".. ... ... ... \n", + "795 50 6 True \n", + "796 110 6 True \n", + "797 70 6 True \n", + "798 80 6 True \n", + "799 70 6 True \n", + "\n", + "[800 rows x 11 columns]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/pokemon.csv\")\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Grass', 'Fire', 'Water', 'Bug', 'Normal', 'Poison', 'Electric',\n", + " 'Ground', 'Fairy', 'Fighting', 'Psychic', 'Rock', 'Ghost', 'Ice',\n", + " 'Dragon', 'Dark', 'Steel', 'Flying'], dtype=object)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"Type 1\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Poison', nan, 'Flying', 'Dragon', 'Ground', 'Fairy', 'Grass',\n", + " 'Fighting', 'Psychic', 'Steel', 'Ice', 'Rock', 'Dark', 'Water',\n", + " 'Electric', 'Fire', 'Ghost', 'Bug', 'Normal'], dtype=object)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"Type 2\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameType 1Type 2HPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
275Mega SceptileGrassDragon7011075145851453False
\n", + "
" + ], + "text/plain": [ + " Name Type 1 Type 2 HP Attack Defense Sp. Atk Sp. Def \\\n", + "275 Mega Sceptile Grass Dragon 70 110 75 145 85 \n", + "\n", + " Speed Generation Legendary \n", + "275 145 3 False " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "both_types = df[(\n", + " (df[\"Type 1\"] == \"Dragon\") |\n", + " (df[\"Type 2\"] == \"Dragon\"))\n", + " &\n", + " ((df[\"Type 1\"] == \"Grass\") |\n", + " (df[\"Type 2\"] == \"Grass\")\n", + ")]\n", + "\n", + "both_types" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index([275], dtype='int64')" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "overlap_pokemon = both_types.index\n", + "overlap_pokemon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- We posit that Pokemons of type Dragon have, on average, more HP stats than Grass. Choose the proper test and, with 5% significance, comment your findings." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# H0: dragon_mean_hp <= grass_mean_hp\n", + "# H1: dragon__mean_hp > grass_mean_hp\n", + "\n", + "alpha = 0.05 #significance level" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7 78\n", + "159 41\n", + "160 61\n", + "161 91\n", + "196 90\n", + "249 75\n", + "360 50\n", + "361 80\n", + "365 75\n", + "366 75\n", + "406 45\n", + "407 65\n", + "408 95\n", + "409 95\n", + "417 80\n", + "418 80\n", + "419 80\n", + "420 80\n", + "425 105\n", + "426 105\n", + "491 58\n", + "492 68\n", + "493 108\n", + "494 108\n", + "540 100\n", + "541 90\n", + "544 150\n", + "545 150\n", + "671 46\n", + "672 66\n", + "673 76\n", + "682 77\n", + "694 52\n", + "695 72\n", + "696 92\n", + "706 100\n", + "707 100\n", + "710 125\n", + "711 125\n", + "712 125\n", + "761 65\n", + "766 58\n", + "767 82\n", + "774 45\n", + "775 68\n", + "776 90\n", + "790 40\n", + "791 85\n", + "794 108\n", + "Name: HP, dtype: int64\n", + "0 45\n", + "1 60\n", + "2 80\n", + "3 80\n", + "48 45\n", + " ..\n", + "783 59\n", + "784 65\n", + "785 55\n", + "786 75\n", + "787 85\n", + "Name: HP, Length: 94, dtype: int64\n" + ] + } + ], + "source": [ + "dragon_hp = df[((df[\"Type 1\"] == \"Dragon\") | (df[\"Type 2\"] == \"Dragon\")) & (~df.index.isin(overlap_pokemon))][\"HP\"]\n", + "grass_hp = df[((df[\"Type 1\"] == \"Grass\") | (df[\"Type 2\"] == \"Grass\")) & (~df.index.isin(overlap_pokemon))][\"HP\"]\n", + "print(dragon_hp)\n", + "print(grass_hp)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(83.16326530612245)" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dragon_hp.mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(66.01063829787235)" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grass_hp.mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.060530308690885e-05\n" + ] + } + ], + "source": [ + "p_value = st.ttest_ind(dragon_hp, grass_hp, equal_var=False, alternative = \"greater\").pvalue\n", + "print(p_value)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "H0 can be rejected\n" + ] + } + ], + "source": [ + "#Decision-making\n", + "if p_value < alpha:\n", + " print(\"H0 can be rejected\")\n", + "else:\n", + " print(\"Fail to reject H0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Since the H0 was rejected, pokemon of the dragon type have statistically significant higher average HP than grass types." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- We posit that Legendary Pokemons have different stats (HP, Attack, Defense, Sp.Atk, Sp.Def, Speed) when comparing with Non-Legendary. Choose the propper test and, with 5% significance, comment your findings.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "# H0: lengendary_pokemons = non-legendary_pokemons\n", + "# H1: lengendary_pokemons != non-legendary_pokemons\n", + "alpha = 0.05" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "HP\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Attack\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Defense\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Sp. Atk\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Sp. Def\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Speed\n", + "P-value: 0.00000\n", + "Reject H0\n" + ] + } + ], + "source": [ + "stats = [\"HP\", \"Attack\", \"Defense\", \"Sp. Atk\", \"Sp. Def\", \"Speed\"]\n", + " \n", + "legendary = df[df[\"Legendary\"] == True]\n", + "non_legendary = df[df[\"Legendary\"] == False]\n", + "\n", + "for stat in stats:\n", + " p_value = st.ttest_ind(\n", + " legendary[stat],\n", + " non_legendary[stat],\n", + " equal_var=False).pvalue\n", + "\n", + " print(f\"\\n{stat}\")\n", + " print(f\"P-value: {p_value:.5f}\")\n", + "\n", + " if p_value < alpha:\n", + " print(\"Reject H0\")\n", + " else:\n", + " print(\"Fail to reject H0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- At the 5% significance level, the p-value for all statistics is less than 0.05. Therefore, we reject the H0 for all six stats. So, there is sufficient statistical evidence that legendary and non-legendary pokemons differ significantly in each of the selected stats." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Challenge 2**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this challenge, we will be working with california-housing data. The data can be found here:\n", + "- https://raw.githubusercontent.com/data-bootcamp-v4/data/main/california_housing.csv" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
longitudelatitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomemedian_house_value
0-114.3134.1915.05612.01283.01015.0472.01.493666900.0
1-114.4734.4019.07650.01901.01129.0463.01.820080100.0
2-114.5633.6917.0720.0174.0333.0117.01.650985700.0
3-114.5733.6414.01501.0337.0515.0226.03.191773400.0
4-114.5733.5720.01454.0326.0624.0262.01.925065500.0
\n", + "
" + ], + "text/plain": [ + " longitude latitude housing_median_age total_rooms total_bedrooms \\\n", + "0 -114.31 34.19 15.0 5612.0 1283.0 \n", + "1 -114.47 34.40 19.0 7650.0 1901.0 \n", + "2 -114.56 33.69 17.0 720.0 174.0 \n", + "3 -114.57 33.64 14.0 1501.0 337.0 \n", + "4 -114.57 33.57 20.0 1454.0 326.0 \n", + "\n", + " population households median_income median_house_value \n", + "0 1015.0 472.0 1.4936 66900.0 \n", + "1 1129.0 463.0 1.8200 80100.0 \n", + "2 333.0 117.0 1.6509 85700.0 \n", + "3 515.0 226.0 3.1917 73400.0 \n", + "4 624.0 262.0 1.9250 65500.0 " + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/california_housing.csv\")\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**We posit that houses close to either a school or a hospital are more expensive.**\n", + "\n", + "- School coordinates (-118, 34)\n", + "- Hospital coordinates (-122, 37)\n", + "\n", + "We consider a house (neighborhood) to be close to a school or hospital if the distance is lower than 0.50.\n", + "\n", + "Hint:\n", + "- Write a function to calculate euclidean distance from each house (neighborhood) to the school and to the hospital.\n", + "- Divide your dataset into houses close and far from either a hospital or school.\n", + "- Choose the propper test and, with 5% significance, comment your findings.\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "# H0: house_close <= house_far\n", + "# H1: house_close > house_far\n", + "\n", + "def euclidean_distance(x1, y1, x2, y2):\n", + " return np.sqrt((x2 - x1)**2 + (y2 - y1)**2)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"distance_school\"] = euclidean_distance(\n", + " df[\"longitude\"],\n", + " df[\"latitude\"],\n", + " -118,\n", + " 34\n", + ")\n", + "\n", + "df[\"distance_hospital\"] = euclidean_distance(\n", + " df[\"longitude\"],\n", + " df[\"latitude\"],\n", + " -122,\n", + " 37\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
longitudelatitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomemedian_house_valuedistance_schooldistance_hospitalclose
0-114.3134.1915.05612.01283.01015.0472.01.493666900.03.6948888.187319False
1-114.4734.4019.07650.01901.01129.0463.01.820080100.03.5525917.966235False
2-114.5633.6917.0720.0174.0333.0117.01.650985700.03.4539408.143077False
3-114.5733.6414.01501.0337.0515.0226.03.191773400.03.4488408.154416False
4-114.5733.5720.01454.0326.0624.0262.01.925065500.03.4568488.183508False
\n", + "
" + ], + "text/plain": [ + " longitude latitude housing_median_age total_rooms total_bedrooms \\\n", + "0 -114.31 34.19 15.0 5612.0 1283.0 \n", + "1 -114.47 34.40 19.0 7650.0 1901.0 \n", + "2 -114.56 33.69 17.0 720.0 174.0 \n", + "3 -114.57 33.64 14.0 1501.0 337.0 \n", + "4 -114.57 33.57 20.0 1454.0 326.0 \n", + "\n", + " population households median_income median_house_value distance_school \\\n", + "0 1015.0 472.0 1.4936 66900.0 3.694888 \n", + "1 1129.0 463.0 1.8200 80100.0 3.552591 \n", + "2 333.0 117.0 1.6509 85700.0 3.453940 \n", + "3 515.0 226.0 3.1917 73400.0 3.448840 \n", + "4 624.0 262.0 1.9250 65500.0 3.456848 \n", + "\n", + " distance_hospital close \n", + "0 8.187319 False \n", + "1 7.966235 False \n", + "2 8.143077 False \n", + "3 8.154416 False \n", + "4 8.183508 False " + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"close\"] = ((df[\"distance_school\"] < 0.5) | (df[\"distance_hospital\"] < 0.5))" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "close_houses = df[df[\"close\"]][\"median_house_value\"]\n", + "far_houses = df[~df[\"close\"]][\"median_house_value\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.5032478884296307e-301\n" + ] + } + ], + "source": [ + "p_value = st.ttest_ind(\n", + " close_houses,\n", + " far_houses,\n", + " equal_var=False,\n", + " alternative=\"greater\").pvalue\n", + "\n", + "print(p_value)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reject H0\n" + ] + } + ], + "source": [ + "alpha = 0.05\n", + "\n", + "if p_value < alpha:\n", + " print(\"Reject H0\")\n", + "else:\n", + " print(\"Fail to reject H0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- There is sufficient statistical evidence to conclude that houses located within 0.50 units of either a school or a hospital have a higher mean house value than houses located farther away." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/lab-hypothesis-testing.ipynb b/lab-hypothesis-testing.ipynb index 0cc26d5..d951be6 100644 --- a/lab-hypothesis-testing.ipynb +++ b/lab-hypothesis-testing.ipynb @@ -45,13 +45,12 @@ "#libraries\n", "import pandas as pd\n", "import scipy.stats as st\n", - "import numpy as np\n", - "\n" + "import numpy as np" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -278,7 +277,7 @@ "[800 rows x 11 columns]" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -288,20 +287,340 @@ "df" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Grass', 'Fire', 'Water', 'Bug', 'Normal', 'Poison', 'Electric',\n", + " 'Ground', 'Fairy', 'Fighting', 'Psychic', 'Rock', 'Ghost', 'Ice',\n", + " 'Dragon', 'Dark', 'Steel', 'Flying'], dtype=object)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"Type 1\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Poison', nan, 'Flying', 'Dragon', 'Ground', 'Fairy', 'Grass',\n", + " 'Fighting', 'Psychic', 'Steel', 'Ice', 'Rock', 'Dark', 'Water',\n", + " 'Electric', 'Fire', 'Ghost', 'Bug', 'Normal'], dtype=object)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"Type 2\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameType 1Type 2HPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
275Mega SceptileGrassDragon7011075145851453False
\n", + "
" + ], + "text/plain": [ + " Name Type 1 Type 2 HP Attack Defense Sp. Atk Sp. Def \\\n", + "275 Mega Sceptile Grass Dragon 70 110 75 145 85 \n", + "\n", + " Speed Generation Legendary \n", + "275 145 3 False " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "both_types = df[(\n", + " (df[\"Type 1\"] == \"Dragon\") |\n", + " (df[\"Type 2\"] == \"Dragon\"))\n", + " &\n", + " ((df[\"Type 1\"] == \"Grass\") |\n", + " (df[\"Type 2\"] == \"Grass\")\n", + ")]\n", + "\n", + "both_types" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index([275], dtype='int64')" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "overlap_pokemon = both_types.index\n", + "overlap_pokemon" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "- We posit that Pokemons of type Dragon have, on average, more HP stats than Grass. Choose the propper test and, with 5% significance, comment your findings." + "- We posit that Pokemons of type Dragon have, on average, more HP stats than Grass. Choose the proper test and, with 5% significance, comment your findings." ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "#code here" + "# H0: dragon_mean_hp <= grass_mean_hp\n", + "# H1: dragon__mean_hp > grass_mean_hp\n", + "\n", + "alpha = 0.05 #significance level" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7 78\n", + "159 41\n", + "160 61\n", + "161 91\n", + "196 90\n", + "249 75\n", + "360 50\n", + "361 80\n", + "365 75\n", + "366 75\n", + "406 45\n", + "407 65\n", + "408 95\n", + "409 95\n", + "417 80\n", + "418 80\n", + "419 80\n", + "420 80\n", + "425 105\n", + "426 105\n", + "491 58\n", + "492 68\n", + "493 108\n", + "494 108\n", + "540 100\n", + "541 90\n", + "544 150\n", + "545 150\n", + "671 46\n", + "672 66\n", + "673 76\n", + "682 77\n", + "694 52\n", + "695 72\n", + "696 92\n", + "706 100\n", + "707 100\n", + "710 125\n", + "711 125\n", + "712 125\n", + "761 65\n", + "766 58\n", + "767 82\n", + "774 45\n", + "775 68\n", + "776 90\n", + "790 40\n", + "791 85\n", + "794 108\n", + "Name: HP, dtype: int64\n", + "0 45\n", + "1 60\n", + "2 80\n", + "3 80\n", + "48 45\n", + " ..\n", + "783 59\n", + "784 65\n", + "785 55\n", + "786 75\n", + "787 85\n", + "Name: HP, Length: 94, dtype: int64\n" + ] + } + ], + "source": [ + "dragon_hp = df[((df[\"Type 1\"] == \"Dragon\") | (df[\"Type 2\"] == \"Dragon\")) & (~df.index.isin(overlap_pokemon))][\"HP\"]\n", + "grass_hp = df[((df[\"Type 1\"] == \"Grass\") | (df[\"Type 2\"] == \"Grass\")) & (~df.index.isin(overlap_pokemon))][\"HP\"]\n", + "print(dragon_hp)\n", + "print(grass_hp)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(83.16326530612245)" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dragon_hp.mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(66.01063829787235)" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grass_hp.mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.060530308690885e-05\n" + ] + } + ], + "source": [ + "p_value = st.ttest_ind(dragon_hp, grass_hp, equal_var=False, alternative = \"greater\").pvalue\n", + "print(p_value)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "H0 can be rejected\n" + ] + } + ], + "source": [ + "#Decision-making\n", + "if p_value < alpha:\n", + " print(\"H0 can be rejected\")\n", + "else:\n", + " print(\"Fail to reject H0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Since the H0 was rejected, pokemon of the dragon type have statistically significant higher average HP than grass types." ] }, { @@ -313,11 +632,77 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ - "#code here" + "# H0: lengendary_pokemons = non-legendary_pokemons\n", + "# H1: lengendary_pokemons != non-legendary_pokemons\n", + "alpha = 0.05" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "HP\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Attack\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Defense\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Sp. Atk\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Sp. Def\n", + "P-value: 0.00000\n", + "Reject H0\n", + "\n", + "Speed\n", + "P-value: 0.00000\n", + "Reject H0\n" + ] + } + ], + "source": [ + "stats = [\"HP\", \"Attack\", \"Defense\", \"Sp. Atk\", \"Sp. Def\", \"Speed\"]\n", + " \n", + "legendary = df[df[\"Legendary\"] == True]\n", + "non_legendary = df[df[\"Legendary\"] == False]\n", + "\n", + "for stat in stats:\n", + " p_value = st.ttest_ind(\n", + " legendary[stat],\n", + " non_legendary[stat],\n", + " equal_var=False).pvalue\n", + "\n", + " print(f\"\\n{stat}\")\n", + " print(f\"P-value: {p_value:.5f}\")\n", + "\n", + " if p_value < alpha:\n", + " print(\"Reject H0\")\n", + " else:\n", + " print(\"Fail to reject H0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- At the 5% significance level, the p-value for all statistics is less than 0.05. Therefore, we reject the H0 for all six stats. So, there is sufficient statistical evidence that legendary and non-legendary pokemons differ significantly in each of the selected stats." ] }, { @@ -337,7 +722,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 48, "metadata": {}, "outputs": [ { @@ -453,7 +838,7 @@ "4 624.0 262.0 1.9250 65500.0 " ] }, - "execution_count": 5, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -483,10 +868,260 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# H0: house_close <= house_far\n", + "# H1: house_close > house_far\n", + "\n", + "def euclidean_distance(x1, y1, x2, y2):\n", + " return np.sqrt((x2 - x1)**2 + (y2 - y1)**2)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"distance_school\"] = euclidean_distance(\n", + " df[\"longitude\"],\n", + " df[\"latitude\"],\n", + " -118,\n", + " 34\n", + ")\n", + "\n", + "df[\"distance_hospital\"] = euclidean_distance(\n", + " df[\"longitude\"],\n", + " df[\"latitude\"],\n", + " -122,\n", + " 37\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
longitudelatitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomemedian_house_valuedistance_schooldistance_hospitalclose
0-114.3134.1915.05612.01283.01015.0472.01.493666900.03.6948888.187319False
1-114.4734.4019.07650.01901.01129.0463.01.820080100.03.5525917.966235False
2-114.5633.6917.0720.0174.0333.0117.01.650985700.03.4539408.143077False
3-114.5733.6414.01501.0337.0515.0226.03.191773400.03.4488408.154416False
4-114.5733.5720.01454.0326.0624.0262.01.925065500.03.4568488.183508False
\n", + "
" + ], + "text/plain": [ + " longitude latitude housing_median_age total_rooms total_bedrooms \\\n", + "0 -114.31 34.19 15.0 5612.0 1283.0 \n", + "1 -114.47 34.40 19.0 7650.0 1901.0 \n", + "2 -114.56 33.69 17.0 720.0 174.0 \n", + "3 -114.57 33.64 14.0 1501.0 337.0 \n", + "4 -114.57 33.57 20.0 1454.0 326.0 \n", + "\n", + " population households median_income median_house_value distance_school \\\n", + "0 1015.0 472.0 1.4936 66900.0 3.694888 \n", + "1 1129.0 463.0 1.8200 80100.0 3.552591 \n", + "2 333.0 117.0 1.6509 85700.0 3.453940 \n", + "3 515.0 226.0 3.1917 73400.0 3.448840 \n", + "4 624.0 262.0 1.9250 65500.0 3.456848 \n", + "\n", + " distance_hospital close \n", + "0 8.187319 False \n", + "1 7.966235 False \n", + "2 8.143077 False \n", + "3 8.154416 False \n", + "4 8.183508 False " + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"close\"] = ((df[\"distance_school\"] < 0.5) | (df[\"distance_hospital\"] < 0.5))" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "close_houses = df[df[\"close\"]][\"median_house_value\"]\n", + "far_houses = df[~df[\"close\"]][\"median_house_value\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.5032478884296307e-301\n" + ] + } + ], + "source": [ + "p_value = st.ttest_ind(\n", + " close_houses,\n", + " far_houses,\n", + " equal_var=False,\n", + " alternative=\"greater\").pvalue\n", + "\n", + "print(p_value)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reject H0\n" + ] + } + ], + "source": [ + "alpha = 0.05\n", + "\n", + "if p_value < alpha:\n", + " print(\"Reject H0\")\n", + "else:\n", + " print(\"Fail to reject H0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- There is sufficient statistical evidence to conclude that houses located within 0.50 units of either a school or a hospital have a higher mean house value than houses located farther away." + ] }, { "cell_type": "code", @@ -498,7 +1133,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -512,9 +1147,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.13.5" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }