.Rhistory

clicksData <- read.csv(file.choose())
clickData
clicksData
clicksData$clicks
boxplot(clicksData$clicks)
boxplot(clicksData$clicks, normal)
norm = pnorm(200)
boxplot(clicksData$clicks, norm)
norm = rnorm(200)
boxplot(norm)
norm
clicks <- clicksData$clicks
xseq <- seq(-4,4,.01)
norm <- pnorm(xseq, mean(clicks), sd(clicks))
plot(norm)
plot(xseq, norm)
norm <- dnorm(xseq, mean(clicks), sd(clicks))
plot(xseq, norm)
plot(norm)
norm <- dnorm(0, mean(clicks), sd(clicks))
plot(norm)
norm <- rnorm(length(clicks), mean(clicks), sd(clicks))
plot(norm)
norm <- dnorm(xseq, mean(clicks), sd(clicks))
plot(xseq, norm)
norm <- dnorm(xseq, 0, 1)
plot(xseq, norm)
xseq <- seq(min(clicks), max(clicks), by=.01)
norm <- dnorm(xseq, mean(clicks), sd(clicks))
plot(xseq, norm)
boxplot(clicks, norm)
boxplot(norm)
boxplot(xsqeq, norm)
boxplot(xseq, norm)
plot(xseq, norm)
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm)
norm <- rnorm(2000, mean(clicks), sd(clicks))
boxplot(clicks, norm)
clicksData <- read.csv(file.choose())
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm)
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm)
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm)
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm, names=c("Clicks", "Normal"))
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
hist(clicks)
hist(clicks, prob=TRUE)
curve(dnorm(xseq, mean(clicks), sd(clicks)), add=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
hist(clicks, prob=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
hist(clicks, prob=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
plot(dnorm(x, mean(clicks), sd(clicks)))
hist(clicks, prob=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
plot(pnorm(p, mean(clicks), sd(clicks)))
xseq <- seq(min(clicks), max(clicks), .01)
plot(pnorm(xseq, mean(clicks), sd(clicks)))
plot(pnorm(xseq, mean(clicks), sd(clicks)), type="l")
lines(xseq, clicks)
curve(clicks, add=TRUE, col="darkblue")
lines(xseq, clicks)
plot(xseq, pnorm(xseq, mean(clicks), sd(clicks)), type="l")
clicks
lines(xseq, clicks)
qqlines(clicks)
qqline(clicks)
qqnorm(clicks)
qqline(clicks)
qqnorm(clicks)
qqline(clicks)
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
hist(clicks, prob=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
qqnorm(clicks)
qqline(clicks)
clicksV1 = clicks[clicks=="variant1"]
clicksV1
clicksV1 <- clicks[clicks=="variant1"]
clicksV1
clicks
clicksV1 <- clicksData[clicksData$clicks=="variant1"]
clicks
clicksV1
clicksV1 <- clicksData[clicksData$clicks=="variant1"]$clicks
clicksV1
clicksV1 <- clicksData[clicksData$clicks=="variant1",]$clicks
clicksV1
clicksV1 <- clicksData[clicksData$cvariant=="variant1",]$clicks
clicksV1
clicksV1 <- clicksData[clicksData$variant=="variant1",]$clicks
clicksV1
shapiro.test(clicks)
shapiro.test(clicks)
shapiro.test(clicksV1)
shapiro.test(clicksV2)
clicksV1 <- clicksData[clicksData$variant=="variant1",]$clicks
clicksV2 <- clicksData[clicksData$variant=="variant2",]$clicks
shapiro.test(clicksV1)
shapiro.test(clicksV2)
size(clicksV1)
length(clicksV1)
length(clicksV2)
t.test(clicksV1, clicksV2, paired=FALSE)
#Import data and select clicks column
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicks <- clicksData$clicks
#We also need some normaly distributed data
norm <- rnorm(200, mean(clicks), sd(clicks))
#draws the boxplot
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
boxplot(clicks, norm, names=c("Clicks", "Normal"), col=c("blue", "red"), ylab="Number of clicks")
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
#draws the histogram
hist(clicks, prob=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
#draws the Q-Q plot
qqnorm(clicks)
qqline(clicks)
var.test(clicksV1, clicksV2)
q()
cpuData
cpuData <- file.select(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question2_4)
cpuData <- file.select("C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question2_4")
cpuData <- read.csv("C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question2_4")
cpuData <- read.csv(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question2_4)
cpuData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question2_4))
cpuData <- read.csv(file.choose("C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question2_4"))
clicksData <- read.csv(file.choose(C:\Users\arnau\Documents\School\Projecten 16_17\Methods_Of_Scientific_Research\Assignment3\Question1_3))
clicksData <- read.csv("Question1_3"))
clicksData <- read.csv("Question1_3")
clicksData <- read.csv("Question1_3.csv")
getwd()
clicksData <- read.csv(file.choose())
cpuData <- read.csv(file.choose())
cpuData
plot(cpuData$temperature)
hist(cpuData$temperature)
hist(cpuData$temperature, cpuData$crashed)
hist(cpuData$crashed)
cpuAov <- aov(temperature~crashed, data=cpuData)
summary(cpuAov)
bar(cpuData$crashed)
 table(cpuData$crashed)
dbinom(1, size=length(cpuData), prob=0.5)
dbinom(41, size=length(cpuData), prob=0.5)
plot(cpuAov)
output<-factor(cpuData$crashed)
output
model<-glm( formula=output~input,family=binomial )
model<-glm( formula=output~temperature,family=binomial )
output<-factor(cpuData)
model<-glm( formula=output~cpuData$temperature,family=binomial )
summary(model)
help("logistics")
dlogis(cpuData)
anova(model, test="Chisq")
logistic(cpuData)
crashed <- cpuData$crashed
temps <- cpuData$temperature
output<-factor(crashed)
model<-glm( formula=output~temps,family=binomial)
summary(model)
plot(temps, output)
points( temps, model$fitted+1, pch=3, col=“red” ))
points(temps, model$fitted+1, pch=3, col=“red”))
plot(temps, output)
points(temps, model$fitted+1, pch=3, col=“red”))
lines(temps, model$fitted+1, pch=3, col=“red”))
lines(temps, model$fitted, pch=3, col=“red”))
model$fitted
lines(temps, model$fitted, pch=3, col=“red”))
lines(temps, model$fitted, pch=3, col=“red”)
lines(temps, model$fitted+1, pch=3, col=“red”)
plot(temps, output)
points(temps, model$fitted+1, pch=3, col=“red”)
points(temps, model$fitted+1, pch=3, col=“red”)
library(ggplot2)
temps
output<-factor(crashed)
model<-glm( formula=output~temps,family=binomial)
summary(model)
plot(temps, output)
points(temps, model$fitted+1, pch=3, col=“red”)
plot(temps, crashed)
plot(model)
plot(temps, model$fitted+1, pch=3, col=“red”)
plot(temps, model$fitted, pch=3, col=“red”)
model$fitted
factor
factor(crashed)
model<-glm( formula=crashed~temps,family=binomial)
summary(model)
plot(temps, crashed)
lines(temps, model$fitted, pch=3, col=“red”)
plot(temps, model$fitted)
output<-factor(crashed)
model<-glm( formula=crashed~temps,family=binomial)
summary(model)
plot(temps, crashed)
lines(temps, model$fitted, pch=3, col=“red”)
lines(temps, model$fitted, pch=3, col="red")
lines(temps, model$fitted+1, pch=3, col="red")
output<-factor(crashed)
model<-glm( formula=crashed~temps,family=binomial)
summary(model)
plot(temps, crashed)
lines(temps, model$fitted+1, pch=3, col="red")
lines(temps, model$fitted, pch=3, col="red")
q()
cpuData <- read.csv(file.choose())
temps <- cpuData$temperature
crashed <- cpuData$crashed
cpuAov <- aov(temperature~crashed, data=cpuData)
summary(cpuAov)
plot(cpuAov)
output<-factor(crashed)
model<-glm( formula=crashed~temps,family=binomial)
summary(model)
plot(temps, crashed)
lines(temps, model$fitted, pch=3, col="red")
points(temps, model$fitted, pch=3, col="red")
plot(temps, crashed)
points(temps, model$fitted, pch=3, col="red")
help(glm)
anova(model)
cpuData
model<-glm( formula=output~temps,family=binomial)
output<-factor(crashed)
model<-glm( formula=output~temps,family=binomial)
summary(model)
plot(temps, crashed)
points(temps, model$fitted, pch=3, col="red")
plot(temps, crashed, ylab="Probability", xlab="Temperature")
points(temps, model$fitted, pch=3, col="red")
help(summary.glm)
progData <- read.csv(file.chose())
progData <- read.csv(file.choose())
progData
model <- lm(formula = progData$score ~ progData$group + progData$age)
summary(model)
model <- lm(formula = scores ~ lang + ages)
scores <- progData$score
lang <- progData$group
ages <- progData$age
model <- lm(formula = scores ~ lang + ages)
plot(scores, ages)
model
model$fit
lines(scores, model$fit)
plot(model)
plot(scores, lang + ages)
plot(scores, lang)
lines(scores, model$fit)
lines(scores, model$fit)
plot(ages, scores)
lines(scores, model$fit)
lines(model$fit, sscores)
lines(model$fit, scores)
points(model$fit, scores)
plot(ages, scores)
points(model$fit, scores)
points(model$fit, scores, col="red")
confint(model)
plot(lang, scores)
plot(lang, scores)
aov(formula = scores ~ lang + ages)
q()
clicksData <- read.csv(file.choose())
clicks <- clicksData$clicks
norm <- rnorm(200, mean(clicks), sd(clicks))
#draws the boxplot
boxplot(clicks, norm, names=c("Clicks", "Normal"), ylab="Number of clicks")
#draws the histogram
hist(clicks, prob=TRUE)
curve(dnorm(x, mean(clicks), sd(clicks)), add=TRUE, col="darkblue")
#draws the Q-Q plot
qqnorm(clicks)
qqline(clicks)
#Test for normality
shapiro.test(clicks)
clicksV1 <- clicksData[clicksData$variant=="variant1",]$clicks
clicksV2 <- clicksData[clicksData$variant=="variant2",]$clicks
shapiro.test(clicksV1)
shapiro.test(clicksV2)
t.test(clicksV1, clicksV2, paired=FALSE)
var.test(clicksV1, clicksV2)
install(pwr)
library(pwr)
help(package)
install.packages(pwr)
install.packages("pwr")
cd <- cohens_d(clicksV1, clicksV2)
#Function to calculate Cohen's d for a t-test
cohens_d <- function(x, y) {
    lx <- length(x)- 1
    ly <- length(y)- 1
    md  <- abs(mean(x) - mean(y))        ## mean difference (numerator)
    csd <- lx * var(x) + ly * var(y)
    csd <- csd/(lx + ly)
    csd <- sqrt(csd)                     ## common sd computation
    cd  <- md/csd                        ## cohen's d
}
cd <- cohens_d(clicksV1, clicksV2)
cd
t.test(clicksV1, clicksV2, paired=FALSE, var.equal=TRUE)
cd <- cohens_d(clicksV1, clicksV2)
cd
pwr.t.test(n = 100, sig.level = 0.05, power = cd, type = "two.sample") 
library(pwr)
pwr.t.test(n = 100, sig.level = 0.05, power = cd, type = "two.sample") 
pwr.t.test(n = 100, sig.level = 0.05, d= cd, type = "two.sample") 
q()
cpuData <- read.csv(file.choose())
temps <- cpuData$temperature
crashed <- cpuData$crashed
output<-factor(crashed)
model<-glm( formula=output~temps,family=binomial)
summary(model)
plot(temps, crashed, ylab="Probability", xlab="Temperature")
points(temps, model$fitted, pch=3, col="red")
progData <- read.csv(file.choose())
pyData <- progData[progData$group=="Python", ]
scData <- progData[progData$group=="Scheme", ]
pyScores <- pyData$score
scScores <- scData$score
shapiro.test(pyScores)
shapiro.test(scScores )
var.test(pyScores , scScores)
t.test(pyScores , scScores , paired=FALSE, var.equal=TRUE)
boxplot(pyScores, scScores, names=c("Python", "Scheme"))
scores <- progData$score
ages <- progData$age
model <- lm(scores~ages)
summary(model)
plot(ages, scores)
abline(model, col="red")
q()
install.packages('devtools')
devtools::install_github('IRkernel/IRkernel')
# or devtools::install_local('IRkernel-master.tar.gz')
devtools::install_local('IRkernel-master.tar.gz')
q()
library(tree)
install.packages(tree)
install.packages("tree")
install.packages("ggplot2")
install.packages("lm")
install.packages("dummies")
setws(C:\Users\arnau\Documents\School\Projecten 16_17\Statistical_Foundations_ML\StatisticalML)
setws("C:\Users\arnau\Documents\School\Projecten 16_17\Statistical_Foundations_ML\StatisticalML")
getws()
getwd
getwd()
setwd(C:\Users\arnau\Documents\School\Projecten 16_17\Statistical_Foundations_ML\StatisticalML)
setwd("C:\Users\arnau\Documents\School\Projecten 16_17\Statistical_Foundations_ML\StatisticalML")
setwd("C:/Users/arnau/Documents/School/Projecten 16_17/Statistical_Foundations_ML/StatisticalML")
seed
source("./code/cross_validation.r")
data.train<-read.csv("./data/train.csv")
X<-data.train[,setdiff(colnames(data.train), "SalePrice")]
Y<-data.train[,"SalePrice]
Y<-data.train[,"SalePrice"]
Y<-data.train[,"SalePrice"]
crossval_nn(X,Y)
X
Y
crossval_nn(X,Y)
source("./code/cross_validation.r")
crossval_nn(X,Y)
crossval_lm(X,Y)
crossval_lm(X,Y)
crossval_nn(X,Y)
crossval_tree(X,Y)
source("./code/cross_validation.r")
crossval_tree(X,Y)
crossval_nn(X,Y)
y_train<-y[k_tr]
crossval_nn <- function(x, y, n_folds=10, size=2, maxit=1000) {
index=sample((1:length(y)))
cv_size<-floor(length(y)/n_folds)
cv_err<-numeric(n_folds)
for (k in 1:n_folds) {
k_ts<-(((k-1)*cv_size + 1) : (k*cv_size))
print(k_ts)
k_tr<-setdiff(index, k_ts)
x_test<-x[k_ts, ]
y_test<-y[k_ts]
x_train<-x[k_tr, ]
y_train<-y[k_tr]
model<-nnet(y_train~., x_train, size=size, maxit=maxit, trace=FALSE, linout=FALSE)
y_hat<-predict(model, x_test)
cv_err[k]<-mean((y_hat-y_test)^2)
}
mean(cv_err)
}
crossval_nn(X,Y)
#Import libraries for tested models
library(nnet)
library(rpart) 
library(ggplot2)
library(dummies)
#Import required scripts
source("./code/feature_selection.r")
source("./code/cross_validation.r")
set.seed(2)
#Import data  (and set working directory if necessary)
#setwd("C:/Users/arnau/Documents/School/Projecten 16_17/Statistical_Foundations_ML/StatisticalML")
data.train<-read.csv("./data/train.csv")
data.test<-read.csv("./data/test.csv")
#Remove categorical variables
factor_variables<-which(sapply(data.train[1,],class)=="factor")
data_preprocessed.train<-data.train[,-factor_variables]
#Replace NA values with the variable mean
data_preprocessed.train<-data.frame(apply(data_preprocessed.train,2,replace_na_with_mean_value))
X.train <- data_preprocessed.train[,setdiff(colnames(data_preprocessed.train), "SalePrice")]
Y.train <- data_preprocessed.train[,"SalePrice",drop=F]
crossvalidate_lm(X.train, Y.train)
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
source("./code/cross_validation.r")
crossval_lm(X.train, Y.train)
q()
gewtd()
getwd()
data.train<-read.csv("./data/train.csv")
data.train[1]
data.train[1,]
data.train[-1,]
prcomp(X.train)
pca<-prcomp(X.train)
summary(pca
)
data.train<-read.csv("./data/train.csv")
data.test<-read.csv("./data/test.csv")
#Remove categorical variables
factor_variables<-which(sapply(data.train[1,],class)=="factor")
data_preprocessed.train<-data.train[,-factor_variables]
#Replace NA values with the variable mean
data_preprocessed.train<-data.frame(apply(data_preprocessed.train,2,replace_na_with_mean_value))
#Add categorical variables again with one-hot-encoding
data_factor.train<-data.train[,factor_variables]
data_factor_onehot.train <- dummy.data.frame(data_factor.train)
data_preprocessed.train<-cbind(data_preprocessed.train, data_factor_onehot.train)
X.train <- data_preprocessed.train[,setdiff(colnames(data_preprocessed.train), "SalePrice")]
Y.train <- data_preprocessed.train[,"SalePrice"]
installpackage(dummies)
install.packages("dummies")
data.train<-read.csv("./data/train.csv")
data.test<-read.csv("./data/test.csv")
#Remove categorical variables
factor_variables<-which(sapply(data.train[1,],class)=="factor")
data_preprocessed.train<-data.train[,-factor_variables]
#Replace NA values with the variable mean
data_preprocessed.train<-data.frame(apply(data_preprocessed.train,2,replace_na_with_mean_value))
#Add categorical variables again with one-hot-encoding
data_factor.train<-data.train[,factor_variables]
data_factor_onehot.train <- dummy.data.frame(data_factor.train)
data_preprocessed.train<-cbind(data_preprocessed.train, data_factor_onehot.train)
X.train <- data_preprocessed.train[,setdiff(colnames(data_preprocessed.train), "SalePrice")]
Y.train <- data_preprocessed.train[,"SalePrice"]
getwd()
q()