#########################################
### Anne-Laure Boulesteix, 11.12.2009 ###
#########################################

library(e1071)
library(limma)
library(randomForest)
library(CMA)


###########################
### Load prostate data ####
###########################

load("prostata.RData")
Y<-as.numeric(prostate[,1])-1
X<-prostate[,-1]
X<-as.matrix(X)


############################
###### learning sets #######
############################

set.seed(111)
lset <- GenerateLearningsets(y=Y, method = "CV", fold=5, strat =TRUE,niter=1)


##########################
########## niter #########
##########################

niter<-20

######################################
########## Defining matrices #########
######################################
methodnames<-c("knn1nb20","knn1nb50","knn1nb100","knn1nb200","knn1nb500",
		"knn3nb20","knn3nb50","knn3nb100","knn3nb200","knn3nb500",
		"knn5nb20","knn5nb50","knn5nb100","knn5nb200","knn5nb500",
		"ldanb10","ldanb20",
		"fdanb10","fdanb20",
		"dldanb20","dldanb50","dldanb100","dldanb200","dldanb500",
		"plslda2nb20","plslda2nb50","plslda2nb100","plslda2nb200","plslda2nb500",
		"plslda3nb20","plslda3nb50","plslda3nb100","plslda3nb200","plslda3nb500",
		"nnetnb20","nnetnb50","nnetnb100","nnetnb200","nnetnb500"
		)

methodnames_all<-c("pam","L2","svm","rfp","rf2p","rf3p","rf4p")


prostateCV_t<-matrix(0,39,niter)
prostateCV_w<-matrix(0,39,niter)
prostateCV_l<-matrix(0,39,niter)
prostateCV_all<-matrix(0,7,niter)

rownames(prostateCV_t)<-methodnames
rownames(prostateCV_w)<-methodnames
rownames(prostateCV_l)<-methodnames
rownames(prostateCV_all)<-methodnames_all


#######################################
############ Main analysis ############
#######################################


permute<-function(y,my.seed)
{
y<-as.factor(y)
set.seed(my.seed)
n<-tapply(y,y,length)

n01<-round(n[1]/sum(n)*n[2])
n00<-n[1]-n01

n11<-n[2]-n01
n10<-n[1]-n00

newy<-numeric(sum(n))
newy[y==levels(y)[1]][sample(n[1],n01)]<-1
newy[y==levels(y)[2]][sample(n[2],n11)]<-1
return(factor(newy))
}



for (i in 1:20)
{
y<-permute(Y,i)


####################################################
##### t-test as preliminary variable selection #####
####################################################

selttest<-GeneSelection(X, y, learningsets = lset, method="t.test")

j<-1
###### knn k=1 ######

prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = selttest, classifier = knnCMA,
                            k = 1, nbgene = 20))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 1, nbgene = 50))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 1, nbgene = 100))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 1, nbgene = 200))@score)
		
j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 1, nbgene = 500))@score)

### knn k=3 ###
j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = selttest, classifier = knnCMA,
                            k = 3, nbgene = 20))@score)


j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 3, nbgene = 50))@score)


j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 3, nbgene = 100))@score)



j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 3, nbgene = 200))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 3, nbgene = 500))@score)

### knn k=5 ###
j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = selttest, classifier = knnCMA,
                            k = 5, nbgene = 20))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 5, nbgene = 50))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 5, nbgene = 100))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 5, nbgene = 200))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = knnCMA,
                            k = 5, nbgene = 500))@score)


### lda ###

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = ldaCMA, nbgene = 10))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = ldaCMA, nbgene = 20))@score)

### fda ###

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = fdaCMA, nbgene = 10))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = fdaCMA, nbgene = 20))@score)

### dlda ###

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = dldaCMA, nbgene = 20))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = dldaCMA, nbgene = 50))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = dldaCMA, nbgene = 100))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = dldaCMA, nbgene = 200))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = dldaCMA, nbgene = 500))@score)

### pls+lda ncomp=2 ###
j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=2, nbgene = 20))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=2, nbgene = 50))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=2, nbgene = 100))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=2, nbgene = 200))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=2, nbgene = 500))@score)

### pls+lda ncomp=3 ###

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=3, nbgene = 20))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=3, nbgene = 50))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=3, nbgene = 100))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=3, nbgene = 200))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selttest, classifier = pls_ldaCMA, comp=3, nbgene = 500))@score)



### nnet ###
j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selttest, classifier = nnetCMA, nbgene=20))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selttest, classifier = nnetCMA, nbgene=50))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selttest, classifier = nnetCMA, nbgene=100))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selttest, classifier = nnetCMA, nbgene=200))@score)

j<-j+1
prostateCV_t[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selttest, classifier = nnetCMA, nbgene=500))@score)


###########################################################
##### Wilcoxon-test as preliminary variable selection #####
###########################################################

selwtest<-GeneSelection(X, y, learningsets = lset, method="wilcox.test")

###### KNN ############

### knn k=1 ###

j<-1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = selwtest, classifier = knnCMA,
                            k = 1, nbgene = 20))@score)
		
j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 1, nbgene = 50))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 1, nbgene = 100))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 1, nbgene = 200))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 1, nbgene = 500))@score)
### knn k=3 ###

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = selwtest, classifier = knnCMA,
                            k = 3, nbgene = 20))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 3, nbgene = 50))@score)



j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 3, nbgene = 100))@score)


j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 3, nbgene = 200))@score)


j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 3, nbgene = 500))@score)

### knn k=5 ###

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = selwtest, classifier = knnCMA,
                            k = 5, nbgene = 20))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 5, nbgene = 50))@score)


j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 5, nbgene = 100))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 5, nbgene = 200))@score)


j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = knnCMA,
                            k = 5, nbgene = 500))@score)

### fda ###
j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = ldaCMA, nbgene = 10))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = ldaCMA, nbgene = 20))@score)


### lda ###

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = fdaCMA, nbgene = 10))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = fdaCMA, nbgene = 20))@score)


### dlda ###

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = dldaCMA, nbgene = 20))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = dldaCMA, nbgene = 50))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = dldaCMA, nbgene = 100))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = dldaCMA, nbgene = 200))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = dldaCMA, nbgene = 500))@score)


### pls+lda ncomp=2 ###


j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=2, nbgene = 20))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=2, nbgene = 50))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=2, nbgene = 100))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=2, nbgene = 200))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=2, nbgene = 500))@score)


### pls+lda ncomp=3 ###

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=3, nbgene = 20))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=3, nbgene = 50))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=3, nbgene = 100))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=3, nbgene = 200))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=selwtest, classifier = pls_ldaCMA, comp=3, nbgene = 500))@score)


### neural networks ###

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selwtest, classifier = nnetCMA, nbgene=20))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selwtest, classifier = nnetCMA, nbgene=50))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selwtest, classifier = nnetCMA, nbgene=100))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selwtest, classifier = nnetCMA, nbgene=200))@score)

j<-j+1
prostateCV_w[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=selwtest, classifier = nnetCMA, nbgene=500))@score)



####################################################
##### limma as preliminary variable selection ######
####################################################

sellimma<-GeneSelection(X, y, learningsets = lset, method="limma")

###### KNN ############

### knn k=1 ###
j<-1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = sellimma, classifier = knnCMA,
                            k = 1, nbgene = 20))@score)
j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 1, nbgene = 50))@score)
j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 1, nbgene = 100))@score)
j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 1, nbgene = 200))@score)
j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 1, nbgene = 500))@score)
### knn k=3 ###
j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = sellimma, classifier = knnCMA,
                            k = 3, nbgene = 20))@score)


j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 3, nbgene = 50))@score)


j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 3, nbgene = 100))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 3, nbgene = 200))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 3, nbgene = 500))@score)

### knn k=5 ###

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel = sellimma, classifier = knnCMA,
                            k = 5, nbgene = 20))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 5, nbgene = 50))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 5, nbgene = 100))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 5, nbgene = 200))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = knnCMA,
                            k = 5, nbgene = 500))@score)

### lda ###

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = ldaCMA, nbgene = 10))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = ldaCMA, nbgene = 20))@score)

### fda ###

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = fdaCMA, nbgene = 10))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = fdaCMA, nbgene = 20))@score)

### dlda ###

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = dldaCMA, nbgene = 20))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = dldaCMA, nbgene = 50))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = dldaCMA, nbgene = 100))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = dldaCMA, nbgene = 200))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = dldaCMA, nbgene = 500))@score)

### pls+lda ncomp=2 ###

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=2, nbgene = 20))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=2, nbgene = 50))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=2, nbgene = 100))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=2, nbgene = 200))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=2, nbgene = 500))@score)


### pls+lda ncomp=3 ###
j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=3, nbgene = 20))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=3, nbgene = 50))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=3, nbgene = 100))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=3, nbgene = 200))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                            genesel=sellimma, classifier = pls_ldaCMA, comp=3, nbgene = 500))@score)

### neural networks ###

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=sellimma, classifier = nnetCMA, nbgene=20))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=sellimma, classifier = nnetCMA, nbgene=50))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=sellimma, classifier = nnetCMA, nbgene=100))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=sellimma, classifier = nnetCMA, nbgene=200))@score)

j<-j+1
prostateCV_l[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       genesel=sellimma, classifier = nnetCMA, nbgene=500))@score)



################################
## Without variable selection ##
################################


j<-1


### PAM ###
tunescda<-tune(X, y, learningsets = lset, classifier = scdaCMA)
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                      tuneres=tunescda,classifier = scdaCMA))@score)

### PLR ###
j<-j+1
tuneplr<-tune(X, y, learningsets = lset, classifier = plrCMA)
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                       tuneres=tuneplr,classifier = plrCMA))@score)

### svm linear ###
j<-j+1
tunesvm<-tune(X, y, learningsets = lset, classifier = svmCMA)
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                      tuneres=tunesvm,classifier = svmCMA))@score)


### rf sqrt(p) ###
j<-j+1
p<-ncol(X)
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                      classifier = rfCMA,mtry=sqrt(p),ntree=1000))@score)

### rf 2sqrt(p) ###
j<-j+1
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                      classifier = rfCMA,mtry=2*sqrt(p),ntree=1000))@score)

### rf 3sqrt(p) ###
j<-j+1
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                      classifier = rfCMA,mtry=3*sqrt(p),ntree=1000))@score)

### rf 4sqrt(p) ###
j<-j+1
prostateCV_all[j,i] <- mean(evaluation(classification(X, y, learningsets = lset,
                      classifier = rfCMA,mtry=4*sqrt(p),ntree=1000))@score)

save(prostateCV_all,file="prostateCV_all.RData")
save(prostateCV_t,file="prostateCV_t.RData")
save(prostateCV_w,file="prostateCV_w.RData")
save(prostateCV_l,file="prostateCV_l.RData")

}