Homec4science
Diffusion HPLnotebooks (master) Browse

HPLnotebooks/HPLstats_tools.py
b894385024cfmaster

Last Change
Raw File

HPLstats_tools.py
View Options

import numpy as np # scientific computation tools
import pandas # data management tools
from statistics import * # standard stats tools
import scipy.stats as stats # advanced stats tools
import researchpy # dataframe stats
import math # basic math
import matplotlib.pyplot as plt # plotting tools
plt.style.use('seaborn-whitegrid') # global style for plotting
from IPython.display import set_matplotlib_formats
set_matplotlib_formats('svg') # high def format from graphs
from IPython.display import display, Math, Markdown # pretty printing in LaTeX
from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets
class HPLstats_tools:
def __init__(self):
# read the data from the iris dataset
self.iris = pandas.read_csv('iris/iris.csv', sep=';', decimal=",", header=0, names=["sepal_length", "sepal_width", "petal_length", "petal_width", "species"])
self.iris['species'] = self.iris['species'].astype('category')
# store iris virginica
self.iris_virginica = self.iris[self.iris.species == 'virginica'].drop(["species"], axis=1)
# get meand and std for petal length of virginica species
self.iv_petal_length_mean = self.iris_virginica["petal_length"].mean()
self.iv_petal_length_std = self.iris_virginica["petal_length"].std()
# sample size
self.sample_size = 50
# welcome message
print("Welcome to the How People Learn notebook on hypothesis testing.")
def get_sample_data(self):
'''The following code generates a pseudo random dataset corresponding to the given sample size.'''
self.sample_data = pandas.DataFrame(
np.random.normal(loc=self.iv_petal_length_mean,scale=self.iv_petal_length_std,size=self.sample_size) + (np.random.rand()/self.iv_petal_length_mean),
columns=["petal_length"])
return self.sample_data
def generate_sample_data(self):
# input
button = widgets.Button(description='Get another sample')
def button_event_handler(b):
self.update()
button.on_click(button_event_handler)
# output
self.output = widgets.Output()
# display interface
display(self.output, button)
# Update it right away
self.update()
def update(self):
# Generation of random data (normal distribution + some small noise)
self.x_data = np.random.normal(loc=self.pop_mean,scale=self.pop_std,size=self.sample_size) + (np.random.rand()/self.pop_mean)
# Let's get some basic descriptive stats
self.x_stats = researchpy.summary_cont(pandas.DataFrame(self.x_data, columns=["Petal Length"]))
x_mean = self.x_stats.at[0,"Mean"]
# Let's have a look at the data
fig = plt.figure(figsize=(16, 4))
# graphical parameters
min_y = 0
max_y = 10
# Values
ax1 = plt.subplot(131)
# ax1.set_ylim(bottom = min_y, top = max_y) # limit bottom of y axis to ground
ax1.set_xlabel('i')
ax1.set_ylabel('x')
ax1.set_title("Samples")
ax1.plot(self.x_data, 'go')
ax1.axhline(y=x_mean, color='black', linestyle='-.', linewidth=1, label="sample mean")
ax1.legend()
# Distribution
ax2 = plt.subplot(132)
# ax2.set_xlim(left = min_y, right = max_y) # limit bottom of y axis to ground
ax2.set_xlabel('x')
ax2.set_ylabel('n')
ax2.set_title("Distribution")
ax2.hist(self.x_data, color='green') # bins=round(sample_size/10),
ax2.axvline(x=x_mean, color='black', linestyle='-.', linewidth=1, label="sample mean")
ax2.legend()
# Box plot
ax3 = plt.subplot(133, sharey = ax1)
box = ax3.boxplot(self.x_data, sym='k+', patch_artist=True)
ax3.set_ylabel('x')
ax3.set_title("Distribution")
ax3.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)
ax3.axhline(y=x_mean, color='black', linestyle='-.', linewidth=1, label="sample mean")
plt.setp(box['medians'], color='black')
for patch in box['boxes']:
patch.set(facecolor='green')
ax3.legend()
# Clear display
self.output.clear_output()
# Update display
with self.output:
# show visualiations
plt.show();
# show stats
display(self.x_stats)
def compare_sample_with_pop(self):
# Let's get the basics stats from the sample
x_n = self.x_stats.at[0,"N"]
x_mean = self.x_stats.at[0,"Mean"]
x_std = self.x_stats.at[0,"SD"]
x_sem = self.x_stats.at[0,"SE"]
### Let's visualize the Z-SCORE
fig = plt.figure(figsize=(16, 4))
# Standard normal distribution
lnspc = np.linspace(-4, 4, self.sample_size)
pdf_norm = stats.norm.pdf(lnspc, 0, 1)
ax1 = plt.subplot(121)
ax1.plot(lnspc, pdf_norm, color='blue', linewidth=1)
ax1.axvline(x=0, color='blue', linestyle='dashed', linewidth=1)
# Confidence zone 5% two tailed
lnspc_zone = np.linspace(stats.norm.ppf(0.025), stats.norm.ppf(0.975), self.sample_size) # 5% confidence two tailed
pdf_norm_zone = stats.norm.pdf(lnspc_zone, 0, 1)
ax1.fill_between(lnspc_zone,pdf_norm_zone,0, alpha=0.3, color='blue')
# Z-Score of the data and p-value
z = (x_mean - self.pop_mean)/(self.pop_std / np.sqrt(self.sample_size))
p = 2*(1-stats.norm.cdf(abs(z))) # two tailed test
ax1.axvline(x=z, color='green', linestyle='dashed', linewidth=1)
ax1.annotate('z-score={:.2f}\np-value={:f}'.format(z, p), xy=(z, 0), xytext=(-10, 100), textcoords='offset points', bbox=dict(boxstyle="round", facecolor = "white", edgecolor = "green", alpha = 0.8))
# Alternatively, t-test
# t, p = stats.ttest_1samp(x, pop_mean)
# ax1.axvline(x=t, color='purple', linestyle='-.', linewidth=1)
### How does that look like on a mean plot with 95 confidence interval
confidence = 0.95
pop_h = (self.pop_std / math.sqrt(self.sample_size)) * 1.96
x_h = x_sem * stats.t.ppf((1 + confidence) / 2, x_n - 1)
ax2 = plt.subplot(122)
ax2.errorbar(x = ['population'], y = [self.pop_mean], yerr = [pop_h], fmt = 'o', capsize=10, color = 'blue')
ax2.errorbar(x = ['sample'], y = [x_mean], yerr = [x_h], fmt = 'o', capsize=10, color = 'green')
plt.show()
c4science · Help