```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

#  This is an accompaniment to the lecture on correlation and difference in Week 3 
##  It includes all the examples used in the lecture


 
### Correlation

```{r}
#We are using a .dat file (survey.dat) created from the SPSS file survey.sav  taken from SPSS Survival Manual 6th Edition Julie Pallant
#http://spss.allenandunwin.com.s3-website-ap-southeast-2.amazonaws.com/data-files.html#.Wb0vvnWP-po
#Results on a survey on well being
#We need to load the file so that we can use it in R.
survey <- read.table("~/workspace/github.com/bambrozio/academic-corner/dit/MScDataAnalytics/probabilityAndStatisticalInference/w3/survey.dat")
#Setting the column names to be that used in the dataset
colnames(survey) <- tolower(colnames(survey))

#Inspecting the data
#Remember to install these packages if you haven't already done so
install.packages("pastecs")
library(pastecs) #For creating descriptive statistic summaries
library(ggplot2) #For creating histograms with more detail than plot
library(psych) # Some useful descriptive functions
library(semTools) #For skewness and kurtosis
library(car) # For Levene's test for homogeneity of variance


#numerical summary and histograms of variables of interest
#Starting with feeling of control
#stat.desc is a function from pastecs - make sure you include the basic switch=F to ensure you don't get scienfitic notation
pastecs::stat.desc(survey$tpcoiss, basic=F)

#skewness and kurtosis from semTools with standard error 
tpskew<-semTools::skew(survey$tpcoiss)
tpkurt<-semTools::kurtosis(survey$tpcoiss)
#We divide the skew statistic by the standard error to get the standardised score
tpskew[1]/tpskew[2]
tpkurt[1]/tpkurt[2]





#We will allocate the histogram to a variable to allow use to manipulate it
gg <- ggplot(survey, aes(x=survey$tpcoiss))

#Change the label of the x axis
gg <- gg + labs(x="Feeling of Control")
#manage binwidth and colours
gg <- gg + geom_histogram(binwidth=2, colour="black", aes(y=..density.., fill=..count..))
gg <- gg + scale_fill_gradient("Count", low="#DCDCDC", high="#7C7C7C")
#adding a normal curve
#use stat_function to compute a normalised score for each value of tpcois
#pass the mean and standard deviation
#use the na.rm parameter to say how missing values are handled
gg <- gg + stat_function(fun=dnorm, color="red",args=list(mean=mean(survey$tpcoiss, na.rm=TRUE), sd=sd(survey$tpcoiss, na.rm=TRUE)))
#to display the graph request the contents of the variable be shown
gg
#Create a qqplot
qqnorm(survey$tpcoiss)
qqline(survey$tpcoiss, col=2) #show a line on theplot




#Create standardised scores and sort
sort(scale(survey$tpcoiss))


#Repeat for the variable representing feelings of stress
pastecs::stat.desc(survey$tpstress, basic=F)
semTools::skew(survey$tpstress)
semTools::kurtosis(survey$tpstress)
gs <- ggplot(survey, aes(x=survey$tpstress))
gs <- gs + labs(x="Perceived Stress")
gs <- gs + geom_histogram(binwidth=2, colour="black", aes(y=..density.., fill=..count..))
gs <- gs + scale_fill_gradient("Count", low="#DCDCDC", high="#7C7C7C")
gs <- gs + stat_function(fun=dnorm, color="red",args=list(mean=mean(survey$tpstress, na.rm=TRUE), sd=sd(survey$tpstress, na.rm=TRUE)))
gs


#Create a qqplot
qqnorm(survey$tpstress)
qqline(survey$tpstress, col=2) #show a line on theplot


#Sort the standardised scores
sort(scale(survey$tpstress))

#Simple scatterplot of feeling of control and perceived stress
#aes(x,y)
scatter <- ggplot(survey, aes(survey$tpstress, survey$tpcoiss))
scatter + geom_point() + labs(x = "Total Perceived Stress", y = "Total PCOISS") 

#Add a regression line
scatter + geom_point() + geom_smooth(method = "lm", colour = "Red", se = F) + labs(x = "Total Perceived Stress", y = "Total PCOISS") 
```

### Conducting Correlation Tests - Pearson, Spearman, Kendall
 

```{r}
#Pearson Correlation
stats::cor.test(survey$tpcoiss, survey$tpstress, method='pearson')

#Spearman Correlation
#Change the method to be spearman.
#This test will give an error since this method uses ranking but cannot handle ties
cor.test(survey$tpcoiss, survey$tpstress, method = "spearman")

#We can also use kendall's tau which does handle ties
cor.test(survey$tpcoiss, survey$tpstress, method = "kendall")
```

### Differences - Parametric Tests
#### Independent t-test
```{r}
#Get descriptive stastitics by group
#describeBy is part of the psych package so you need to use it
psych::describeBy(survey$tslfest,group=survey$sex)

#Conduct Levene's test for homogeneity of variance in library car
car::leveneTest(tslfest ~ sex, data=survey)

#Conduct the t-test from package stats
#You can use the var.equal = TRUE option to specify equal variances and a pooled variance estimate
stats::t.test(tslfest~sex,var.equal=TRUE,data=survey)

```
### Differences - Non-Parametric Tests
#### Mann Whitney U Test
```{r}
drinkset<- read.table("C:/tempR/Field-BDI-Non-parametric.dat")
#Get your descriptive statistcs
describeBy(drinkset$bdisun,group=drinkset$drink)
#Create data subsets for each drink
vodkadata <- subset(drinkset, drink=1)
beerdata <-subset(drinkset, drink=2)

#Create plots of these
gs <- ggplot(vodkadata, aes(x=vodkadata$bdisun))
gs <- gs + labs(x="Vodka")
gs <- gs + geom_histogram(binwidth=2, colour="black", aes(y=..density.., fill=..count..))
gs <- gs + scale_fill_gradient("Count", low="#DCDCDC", high="#7C7C7C")
gs <- gs + stat_function(fun=dnorm, color="red",args=list(mean=mean(vodkadata$bdisun, na.rm=TRUE), sd=sd(vodkadata$bdisun, na.rm=TRUE)))
gs

gs <- ggplot(beerdata, aes(x=beerdata$bdisun))
gs <- gs + labs(x="Beer")
gs <- gs + geom_histogram(binwidth=2, colour="black", aes(y=..density.., fill=..count..))
gs <- gs + scale_fill_gradient("Count", low="#DCDCDC", high="#7C7C7C")
gs <- gs + stat_function(fun=dnorm, color="red",args=list(mean=mean(beerdata$bdisun, na.rm=TRUE), sd=sd(beerdata$bdisun, na.rm=TRUE)))
gs

# Test for differences on  Sunday
stats::wilcox.test(bdisun~drink, data=drinkset) 
# Test for differences on Wednesday
wilcox.test(bdiwed~drink, data=drinkset) 
```