📊 Statistics for Data Science

# --- Code cell 1 ---
import warnings
warnings.filterwarnings("ignore")

# import modules
import numpy as np
import pandas as pd

# plotting modules
import seaborn as sns
import matplotlib.pyplot as plt

# --- Code cell 2 ---
from IPython.core.display import HTML

HTML("""
<style>
h1 { color: Purple !important; }
h2 { color: green !important; }
h3 { color: blue !important; }
</style>
""")

# --- Code cell 6 ---
np.random.seed(10)
original_data = np.random.normal(size = 10000)  # by default mean is zero and variance is 1

# --- Code cell 7 ---
len(original_data)

# --- Code cell 8 ---
# creating axes to draw plots
fig, ax = plt.subplots(1, 2,figsize = (30,10))
 
sns.distplot(original_data, hist = False, kde = True,
            kde_kws = {'shade': True, 'linewidth': 2},
            color ="green", ax = ax[0])

sns.distplot(original_data, hist = True, kde = False,
            kde_kws = {'shade': True, 'linewidth': 2},
             color ="green", ax = ax[1])
 
# rescaling the subplots
fig.set_figheight(5)
fig.set_figwidth(10)

# --- Code cell 13 ---
#conda install xlrd  / pip install xlrd  ( in anconda comand prompt)
data = pd.read_excel("titanic3.xlsx")

# --- Code cell 14 ---
data.head(10)

# --- Code cell 15 ---
print(data.isnull().sum()) # add example of percentage

# --- Code cell 16 ---
print(data.isnull().sum()*100/len(data))

# --- Code cell 17 ---
#Drop columns with IDs and large number of missing values
data.drop(["name", "ticket", "cabin","boat","body","home.dest"],axis=1,inplace=True)

# --- Code cell 18 ---
data.describe()

# --- Code cell 19 ---
data.describe(include='all')

# --- Code cell 20 ---
data['embarked'].value_counts()

# --- Code cell 21 ---
data['embarked'].fillna('S',inplace=True)

# --- Code cell 22 ---
data['embarked'].value_counts()

# --- Code cell 23 ---
import statistics
age_variance = statistics.variance(data['age'].dropna())
print("Variance of Age: ", age_variance)

# --- Code cell 24 ---
std_dev_age = statistics.sqrt(age_variance)
print("Standard deviation of Age: ", std_dev_age)

# --- Code cell 25 ---

sns.distplot(data['age'], hist = True, kde = False,color ="green")

# --- Code cell 27 ---
print("mean age before imputation: ", data['age'].mean())
#print("\n")
data['age'].fillna(data['age'].median(),inplace=True)
print("mean age after imputation: ", data['age'].mean())
print(data.isnull().sum())

# --- Code cell 28 ---
sns.distplot(data['age'], hist = True, kde = False,color ="green")

# --- Code cell 29 ---
#Another options:
    
    #Drop columns with missing data  60%
    #Fit regresison model to predict missing age data
    # Always check accuracy of main task after imputation 
    # Take business logic into consideration for data imputation

# --- Code cell 35 ---
data = pd.read_csv("Housing.csv")

# --- Code cell 36 ---
data = data[['price','area']]
data.describe()

# --- Code cell 37 ---
sns.boxplot(data['area']).set(xlabel= 'area')

# --- Code cell 38 ---
# outlier treatment for area
# data['area']  / data.area
Q1 = data.area.quantile(0.25)
Q3 = data.area.quantile(0.75)
IQR = Q3 - Q1

print("Q1 value: ", Q1)
print("Q3 value: ", Q3)
print("IQR value: ", Q3 - Q1)

# --- Code cell 39 ---
print("Lower threshold of outlier value: ", Q1 - 1.5*IQR)
print("Upper threshold of outlier value: ", Q3 + 1.5*IQR)

# --- Code cell 40 ---
#3650

# --- Code cell 41 ---
data = data[(data.area >= Q1 - 1.5*IQR) & (data.area <= Q3 + 1.5*IQR)]
data.describe()

# --- Code cell 44 ---
data.describe()

# --- Code cell 46 ---
# 0 to 1 scaling
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
data_scaled = pd.DataFrame(scaler.fit_transform(data))
data_scaled.columns = ['price','area']
data_scaled.head(10)

# --- Code cell 47 ---
data_scaled.describe()

# --- Code cell 50 ---
data.describe()

# --- Code cell 52 ---
# z score scaling
#  0 mean and variance /standard deviation 1 

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
data_scaled = pd.DataFrame(scaler.fit_transform(data))
data_scaled.columns = ['price','area']
data_scaled.head(10)

# --- Code cell 53 ---
# if you note - mean will is zero and standard deviation is 1. 
# Min and max vlaue is based on the data after applying transformation 
data_scaled.describe()

# --- Code cell 56 ---
data.head(10)

# --- Code cell 57 ---
sns.scatterplot(data=data, x="area", y="price").set(xlabel= "area",ylabel="price")

# --- Code cell 58 ---
#Correlation of output with numerical variables

# plotting correlation heatmap
dataplot = sns.heatmap(data[['price', 'area']].corr(), cmap="YlGnBu", annot=True)
  
# displaying heatmap
plt.show()