R version 2.6.0 (2007-10-03) Copyright (C) 2007 The R Foundation for Statistical Computing ISBN 3-900051-07-0 R is free software and comes with ABSOLUTELY NO WARRANTY. You are welcome to redistribute it under certain conditions. Type 'license()' or 'licence()' for distribution details. R is a collaborative project with many contributors. Type 'contributors()' for more information and 'citation()' on how to cite R or R packages in publications. Type 'demo()' for some demos, 'help()' for on-line help, or 'help.start()' for an HTML browser interface to help. Type 'q()' to quit R. [Previously saved workspace restored] > # STA 414, 2012, ASSIGNMENT 2 SOLUTION, SCRIPT FOR DATA SET 1. > # > # Radford M. Neal, 2012 > > source("mlp.r") > > # Read and scale data. > > trn <- as.matrix (read.table("artc-trn.dat",head=TRUE)) > test <- as.matrix (read.table("artc-test.dat",head=TRUE)) > > test.y <- test[,1] > trn.y <- trn[,1] > > trn.x <- scale (trn[,-1]) > test.x <- scale (test[,-1], center = attr(trn.x,"scaled:center"), + scale = attr(trn.x,"scaled:scale")) > > # Print performance just predicting the mean of the training responses. > > cat("Average test log probability with training mean:", + mean(log(ifelse(test.y==1,mean(trn.y),1-mean(trn.y)))), "\n\n") Average test log probability with training mean: -0.5875961 > > # Learning constants. > > iters <- 10000 > m <- 6 > S <- 4 > > # Do runs with no stepsize adjustment. > > pdf("artc-noadj.pdf",width=6,height=6) > par(mfrow=c(2,2)) > > cat("No stepsize adjustment:\n\n") No stepsize adjustment: > set.seed(1) > r.noadj <- mlp.cross.val (trn.y, trn.x, eta1=0.0003, eta2=0.0003, + try1=c(0,1,2,4,8,16), try2=0, rel.eta=FALSE, + S=S, iters=iters, m=m, Xtst=test.x, ytst=test.y, + cv.plot=TRUE) --- Lambda parameters: 0 0 --- Best validation log prob at iteration 3193 : -0.5151 Best validation log prob at iteration 1709 : -0.6233 Best validation log prob at iteration 1 : -0.5842 Best validation log prob at iteration 2228 : -0.5059 Average validation log prob : -0.5571 --- Lambda parameters: 1 0 --- Best validation log prob at iteration 3081 : -0.5088 Best validation log prob at iteration 1697 : -0.6205 Best validation log prob at iteration 2806 : -0.5622 Best validation log prob at iteration 2654 : -0.4929 Average validation log prob : -0.5461 --- Lambda parameters: 2 0 --- Best validation log prob at iteration 6972 : -0.5195 Best validation log prob at iteration 1620 : -0.6239 Best validation log prob at iteration 1 : -0.5843 Best validation log prob at iteration 2916 : -0.4937 Average validation log prob : -0.5553 --- Lambda parameters: 4 0 --- Best validation log prob at iteration 4549 : -0.5099 Best validation log prob at iteration 5001 : -0.5884 Best validation log prob at iteration 3797 : -0.5701 Best validation log prob at iteration 2944 : -0.4931 Average validation log prob : -0.5404 --- Lambda parameters: 8 0 --- Best validation log prob at iteration 4332 : -0.5138 Best validation log prob at iteration 7787 : -0.5497 Best validation log prob at iteration 4995 : -0.5732 Best validation log prob at iteration 2888 : -0.4943 Average validation log prob : -0.5327 --- Lambda parameters: 16 0 --- Best validation log prob at iteration 6550 : -0.5137 Best validation log prob at iteration 7883 : -0.5751 Best validation log prob at iteration 8681 : -0.5771 Best validation log prob at iteration 4894 : -0.4914 Average validation log prob : -0.5393 > r.noadj$best.tst.pred <- NULL > print(r.noadj) $lambda1 [1] 8 $lambda2 [1] 0 $best.val.pr [1] -0.5327297 $val.res lambda2=0 lambda1=0 -0.5571295 lambda1=1 -0.5461008 lambda1=2 -0.5553153 lambda1=4 -0.5403791 lambda1=8 -0.5327297 lambda1=16 -0.5393288 $tst.res lambda2=0 lambda1=0 -0.5512848 lambda1=1 -0.5564211 lambda1=2 -0.5579467 lambda1=4 -0.5595363 lambda1=8 -0.5464654 lambda1=16 -0.5419328 > > dev.off() null device 1 > > # Do runs with stepsize adjustment. > > pdf("artc-adj.pdf",width=6,height=6) > par(mfrow=c(2,2)) > > cat("\nWith stepsize adjustment:\n\n") With stepsize adjustment: > set.seed(1) > r.adj <- mlp.cross.val (trn.y, trn.x, eta1=0.0003, eta2=0.0003, + try1=c(0,1,2,4,8,16), try2=0, rel.eta=TRUE, + S=S, iters=iters, m=m, Xtst=test.x, ytst=test.y, + cv.plot=TRUE) --- Lambda parameters: 0 0 --- Best validation log prob at iteration 3350 : -0.5194 Best validation log prob at iteration 2114 : -0.6011 Best validation log prob at iteration 432 : -0.5837 Best validation log prob at iteration 1823 : -0.4948 Average validation log prob : -0.5497 --- Lambda parameters: 1 0 --- Best validation log prob at iteration 3219 : -0.5204 Best validation log prob at iteration 2134 : -0.6024 Best validation log prob at iteration 310 : -0.5839 Best validation log prob at iteration 3586 : -0.4987 Average validation log prob : -0.5514 --- Lambda parameters: 2 0 --- Best validation log prob at iteration 3055 : -0.5182 Best validation log prob at iteration 5910 : -0.5813 Best validation log prob at iteration 372 : -0.5841 Best validation log prob at iteration 3325 : -0.4956 Average validation log prob : -0.5448 --- Lambda parameters: 4 0 --- Best validation log prob at iteration 4877 : -0.5153 Best validation log prob at iteration 8048 : -0.5575 Best validation log prob at iteration 5154 : -0.5836 Best validation log prob at iteration 4257 : -0.4957 Average validation log prob : -0.538 --- Lambda parameters: 8 0 --- Best validation log prob at iteration 5057 : -0.5141 Best validation log prob at iteration 4576 : -0.5911 Best validation log prob at iteration 1 : -0.5843 Best validation log prob at iteration 3469 : -0.4924 Average validation log prob : -0.5455 --- Lambda parameters: 16 0 --- Best validation log prob at iteration 7313 : -0.5144 Best validation log prob at iteration 10000 : -0.5907 Best validation log prob at iteration 111 : -0.5844 Best validation log prob at iteration 7348 : -0.4928 Average validation log prob : -0.5456 > r.adj$best.tst.pred <- NULL > print(r.adj) $lambda1 [1] 4 $lambda2 [1] 0 $best.val.pr [1] -0.5380118 $val.res lambda2=0 lambda1=0 -0.5497485 lambda1=1 -0.5513575 lambda1=2 -0.5448073 lambda1=4 -0.5380118 lambda1=8 -0.5454593 lambda1=16 -0.5455760 $tst.res lambda2=0 lambda1=0 -0.5448182 lambda1=1 -0.5428778 lambda1=2 -0.5436005 lambda1=4 -0.5315223 lambda1=8 -0.5425502 lambda1=16 -0.5442177 > > dev.off() null device 1 > > # Compare weight magnitudes over runs with and without stepsize adjustment. > > pdf("artc-cmp.pdf",width=6,height=4) > par(mfrow=c(1,2)) > > n <- nrow(trn.x) > p <- ncol(trn.x) > > wh <- (round(0*n/S)+1):round(1*n/S) > > for (rel.eta in c(FALSE,TRUE)) + { + set.seed(1) + res <- mlp.train (trn.y[wh], trn.x[wh,], eta1=0.0003, eta2=0.0003, + lambda1=0, lambda2=0, rel.eta=rel.eta, iters=iters, m=m) + + wsq <- matrix(NA,iters,p) + wts <- res$params [, (2*m+2):ncol(res$params)] + + for (i in 1:iters) + { wsq[i,] <- rowSums(matrix(wts[i,]^2,p,m)) + } + + plot (c(1,iters), c(1e-4,1e1), type="n", log="y", + xlab="iteration", ylab="log sum of squares of input-hidden weights") + title(paste("rel.eta =",rel.eta)) + for (j in 1:p) + { lines (wsq[,j], col=(j-1)%%5+1, lty=(j-1)%/%5+1) + } + } > > dev.off() null device 1 > > proc.time() user system elapsed 996.035 2.705 998.664