🧪 A/B Testing

import pandas as pd
from scipy import stats

# Load A/B test data (download from link above!)
# This data has conversion rates for 35 days
data = pd.read_csv("AB_testing_data.csv")

# Let's look at the data
print(data.head(10))

# Output:
#    Day  Conversion fraction A  Conversion fraction B
# 0    1                  0.102                  0.189
# 1    2                  0.095                  0.178
# 2    3                  0.108                  0.192
# ...

# Calculate average conversion rate for each version
avg_A = data['Conversion fraction A'].mean()
avg_B = data['Conversion fraction B'].mean()

print(f"Version A average conversion: {avg_A:.1%}")
print(f"Version B average conversion: {avg_B:.1%}")
print(f"Difference: {(avg_B - avg_A):.1%}")

# Output:
# Version A average conversion: 10.2%
# Version B average conversion: 18.5%
# Difference: 8.3%

# Wow! B looks much better! But is it STATISTICALLY significant?

# Get the conversion rates for each version
group_A = data['Conversion fraction A']
group_B = data['Conversion fraction B']

# Run the T-Test!
t_stat, p_value = stats.ttest_ind(group_A, group_B)

print("=" * 50)
print("       A/B TEST RESULTS")
print("=" * 50)
print(f"T-statistic: {t_stat:.2f}")
print(f"P-value: {p_value:.6f}")
print("=" * 50)

# Output:
# ==================================================
#        A/B TEST RESULTS
# ==================================================
# T-statistic: -3.74
# P-value: 0.000347
# ==================================================

# Make a decision based on p-value
alpha = 0.05  # Significance threshold (5%)

if p_value < alpha:
    print("✅ STATISTICALLY SIGNIFICANT!")
    print("The difference is REAL, not random luck.")
    print(f"We are {(1 - p_value) * 100:.2f}% confident Version B is better!")
    print("\n👉 RECOMMENDATION: Implement Version B!")
else:
    print("❌ NOT statistically significant.")
    print("The difference might just be random chance.")
    print("\n👉 RECOMMENDATION: Keep Version A or test longer.")

# Output:
# ✅ STATISTICALLY SIGNIFICANT!
# The difference is REAL, not random luck.
# We are 99.97% confident Version B is better!
#
# 👉 RECOMMENDATION: Implement Version B!

# --- Code cell 1 ---
from IPython.core.display import HTML

HTML("""
<style>

h2 { color: blue !important; }
h3 { color: green !important; }
</style>
""")

# --- Code cell 4 ---
import pandas as pd
from scipy import stats

# --- Code cell 5 ---
data = pd.read_csv("AB_testing_data.csv")

# --- Code cell 6 ---
len(data)

# --- Code cell 7 ---
data.head(10)

# --- Code cell 8 ---
data.info()

# --- Code cell 9 ---
data.describe()

# --- Code cell 11 ---
samples_set1 = data['Conversion fraction A']
samples_set2 = data['Conversion fraction B']
stat, p = stats.ttest_ind(samples_set1, samples_set2,equal_var = True)

print("AB test results: ")
print("p-value : ", p)
print("")
print("")

# --- Code cell 12 ---
1-0.00034704350989135126

# --- Code cell 13 ---
1-0.05

# --- Code cell 14 ---
# p value < 0.05 so two versions of website have different means for conversion rate - more than 95% confidence