> ################################################################################## > # AnalyzeStudy7.txt > ################################################################################## > # Analysis of small sample size data from Study 7, correlated effect size and sample size, > # Beta effect size, Poisson regression for sample size, F-tests with df1=1 > > rm(list=ls()) > options(scipen=999) # To avoid scientific notation! > # read.table chokes on large data sets > simdata = scan("Study7.Data.txt") Read 6793200 items > # simdata = scan("Study4.5.Data.txt") > # Calculate the number of simulations. It might not be exactly 10,000. There are > # 5 values of Corr(X,N) after selection > # 5 values of k = Number of studies > # 3 values of mean power after selection > # 9 data values per line, for a total of > # 5*5*3*9 = 675 numbers produced by each simulation. > nsim = length(simdata)/675 > cat("\nAnalyzing data from",nsim," simulations \n\n") Analyzing data from 10064 simulations > > nrows = length(simdata)/9; ncols = 9 > dim(simdata) = c(ncols,nrows) # Reversed: R reads by columns, rows > simdata = t(simdata) > colnames(simdata) = c("Simulation", "Correlation", "Ntests", "PopMeanPower", + "SampleMeanPower", "Pcurve", "Puniform", "MaxLike", "Zcurve") > head(simdata) ; tail(simdata) Simulation Correlation Ntests PopMeanPower SampleMeanPower Pcurve Puniform MaxLike Zcurve [1,] 1 0 100 0.25 0.2344754 0.3332208 0.8470190 0.2830611 0.2447621 [2,] 1 0 100 0.50 0.4507504 0.7249105 0.8949652 0.4934986 0.4729016 [3,] 1 0 100 0.75 0.7393955 0.9920565 0.9716985 0.8641121 0.8334374 [4,] 1 0 250 0.25 0.2478505 0.3664215 0.8498973 0.2838998 0.2105447 [5,] 1 0 250 0.50 0.4952208 0.7809601 0.8926206 0.4792957 0.4417398 [6,] 1 0 250 0.75 0.7580721 0.9899538 0.9654522 0.8322289 0.8150287 Simulation Correlation Ntests PopMeanPower SampleMeanPower Pcurve Puniform MaxLike Zcurve [754795,] 35 -0.8 1000 0.25 0.2546670 0.4159138 0.8534462 0.3073517 0.2194036 [754796,] 35 -0.8 1000 0.50 0.4933359 0.8234132 0.9031733 0.5152731 0.4809751 [754797,] 35 -0.8 1000 0.75 0.7428861 0.9898861 0.9639059 0.8210291 0.7846441 [754798,] 35 -0.8 2000 0.25 0.2593913 0.4146790 0.8535184 0.3046363 0.2378361 [754799,] 35 -0.8 2000 0.50 0.4874926 0.8185314 0.9038836 0.5209267 0.4851085 [754800,] 35 -0.8 2000 0.75 0.7333417 0.9874516 0.9622571 0.8198683 0.7647143 > > simdata[,8][simdata[,8]>1] = NA # Just in case > # Assembing data in univariate format for nice easy tables > # This way is fast. > nlines = dim(simdata)[1] # Will be bigger for full data set. > # Four copies of the IV columns > Correlation = rep(simdata[,2],each=4) > Ntests = rep(simdata[,3],each=4) > PopMeanPower = rep(simdata[,4],each=4) > SampleMeanPower = rep(simdata[,5],each=4) > Methods = c("Pcurve","Puniform","MaxLike","Zcurve") > Method = factor(rep(1:4,times=nlines),labels=Methods) > est = as.matrix(simdata[,6:9]) # Estimates > Estimate = as.numeric(t(est)) > # cbind(Correlation, Ntests, PopMeanPower, SampleMeanPower, Estimate)[1:8,] > Bias = Estimate - PopMeanPower > AbsError = abs(Bias) > # Call the estimate close enough if it's within 5% of the target. > CloseEnough = numeric(length(AbsError)); CloseEnough[AbsError<0.05] = 1 > > > > ######################### Sample sizes ############################# > kounts = table(Correlation, Method, PopMeanPower,Ntests) > cat("\n Sample sizes \n\n"); print(kounts) Sample sizes , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10064 10064 10064 10064 -0.6 10064 10064 10064 10064 -0.4 10064 10064 10064 10064 -0.2 10064 10064 10064 10064 0 10064 10064 10064 10064 > > > ###################### Average Performance ###################### > estframe = aggregate(Estimate,by=list(Correlation, Method, PopMeanPower,Ntests),FUN=mean, na.rm=T) > meanest = estframe$x; dim(meanest) = c(5,4,3,5) # Numbers of factor levels, in order > MeanEstimatedPower = kounts # To use the nice labels > for(j in 1:5) MeanEstimatedPower[,,,j] = meanest[,,,j] > cat("\n Mean Estimated Power \n\n") Mean Estimated Power > print(round(MeanEstimatedPower,3)) , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.373 0.853 0.301 0.247 -0.6 0.369 0.852 0.299 0.245 -0.4 0.366 0.852 0.299 0.245 -0.2 0.368 0.852 0.299 0.245 0 0.367 0.852 0.298 0.244 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.821 0.906 0.533 0.532 -0.6 0.823 0.906 0.535 0.534 -0.4 0.822 0.906 0.535 0.534 -0.2 0.822 0.906 0.535 0.535 0 0.824 0.907 0.537 0.537 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.988 0.964 0.822 0.820 -0.6 0.989 0.965 0.828 0.824 -0.4 0.989 0.966 0.829 0.825 -0.2 0.990 0.966 0.833 0.827 0 0.990 0.967 0.835 0.829 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.391 0.853 0.301 0.237 -0.6 0.389 0.852 0.300 0.236 -0.4 0.387 0.852 0.300 0.236 -0.2 0.387 0.852 0.299 0.235 0 0.387 0.852 0.299 0.235 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.832 0.906 0.533 0.512 -0.6 0.833 0.906 0.534 0.514 -0.4 0.834 0.906 0.534 0.514 -0.2 0.834 0.906 0.535 0.515 0 0.835 0.906 0.534 0.515 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.990 0.964 0.825 0.808 -0.6 0.991 0.966 0.831 0.812 -0.4 0.991 0.966 0.833 0.814 -0.2 0.991 0.967 0.836 0.816 0 0.992 0.967 0.838 0.817 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.401 0.853 0.301 0.234 -0.6 0.398 0.852 0.300 0.233 -0.4 0.397 0.852 0.300 0.232 -0.2 0.397 0.852 0.300 0.232 0 0.396 0.852 0.300 0.232 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.836 0.906 0.532 0.501 -0.6 0.838 0.906 0.533 0.502 -0.4 0.838 0.906 0.533 0.502 -0.2 0.839 0.906 0.534 0.503 0 0.839 0.906 0.535 0.504 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.990 0.964 0.825 0.796 -0.6 0.991 0.966 0.832 0.801 -0.4 0.992 0.966 0.835 0.804 -0.2 0.992 0.967 0.837 0.805 0 0.992 0.967 0.839 0.807 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.407 0.853 0.302 0.232 -0.6 0.405 0.852 0.301 0.231 -0.4 0.403 0.852 0.300 0.230 -0.2 0.403 0.852 0.300 0.231 0 0.402 0.852 0.300 0.230 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.839 0.906 0.532 0.493 -0.6 0.840 0.906 0.533 0.494 -0.4 0.841 0.906 0.533 0.495 -0.2 0.841 0.906 0.534 0.495 0 0.841 0.906 0.534 0.495 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.990 0.964 0.826 0.785 -0.6 0.991 0.966 0.832 0.790 -0.4 0.992 0.966 0.836 0.793 -0.2 0.992 0.967 0.838 0.794 0 0.992 0.967 0.840 0.796 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.411 0.853 0.302 0.232 -0.6 0.409 0.852 0.302 0.231 -0.4 0.408 0.852 0.301 0.230 -0.2 0.407 0.852 0.301 0.230 0 0.407 0.852 0.301 0.230 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.840 0.906 0.532 0.487 -0.6 0.841 0.906 0.533 0.488 -0.4 0.842 0.906 0.533 0.489 -0.2 0.842 0.906 0.534 0.489 0 0.842 0.906 0.534 0.490 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.991 0.964 0.827 0.774 -0.6 0.991 0.966 0.833 0.779 -0.4 0.992 0.966 0.836 0.781 -0.2 0.992 0.967 0.838 0.783 0 0.993 0.967 0.840 0.785 > > > ########################## Standard deviations ######################### > vframe = aggregate(Estimate,by=list(Correlation, Method, PopMeanPower,Ntests),FUN=var, na.rm=T) > vest = vframe$x; dim(vest) = c(5,4,3,5) # Numbers of factor levels, in order > SDestimatedPower = kounts # To use the nice labels > for(j in 1:5) SDestimatedPower[,,,j] = sqrt(vest[,,,j]) > cat("\n Standard deviations of estimated power \n\n") Standard deviations of estimated power > print(round(SDestimatedPower,3)) , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.131 0.011 0.043 0.057 -0.6 0.131 0.011 0.043 0.057 -0.4 0.131 0.011 0.043 0.057 -0.2 0.133 0.011 0.043 0.057 0 0.130 0.011 0.043 0.056 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.077 0.013 0.057 0.075 -0.6 0.074 0.012 0.056 0.075 -0.4 0.074 0.012 0.056 0.074 -0.2 0.075 0.013 0.056 0.075 0 0.074 0.013 0.057 0.074 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.009 0.010 0.052 0.034 -0.6 0.009 0.010 0.051 0.033 -0.4 0.009 0.010 0.050 0.032 -0.2 0.008 0.010 0.049 0.031 0 0.008 0.009 0.049 0.031 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.087 0.007 0.027 0.037 -0.6 0.087 0.007 0.028 0.037 -0.4 0.087 0.007 0.028 0.037 -0.2 0.085 0.007 0.028 0.037 0 0.087 0.007 0.028 0.037 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.046 0.008 0.036 0.047 -0.6 0.046 0.008 0.036 0.047 -0.4 0.046 0.008 0.036 0.047 -0.2 0.046 0.008 0.036 0.047 0 0.045 0.008 0.036 0.047 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.005 0.006 0.032 0.023 -0.6 0.005 0.006 0.031 0.022 -0.4 0.005 0.006 0.031 0.022 -0.2 0.004 0.006 0.031 0.021 0 0.004 0.006 0.030 0.021 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.061 0.005 0.020 0.028 -0.6 0.062 0.005 0.020 0.028 -0.4 0.061 0.005 0.021 0.028 -0.2 0.063 0.005 0.021 0.028 0 0.062 0.005 0.020 0.028 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.032 0.006 0.028 0.033 -0.6 0.032 0.006 0.026 0.033 -0.4 0.031 0.006 0.027 0.033 -0.2 0.031 0.006 0.026 0.033 0 0.031 0.006 0.025 0.033 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.003 0.004 0.023 0.018 -0.6 0.003 0.004 0.022 0.017 -0.4 0.003 0.004 0.022 0.016 -0.2 0.003 0.004 0.021 0.016 0 0.003 0.004 0.021 0.016 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.043 0.003 0.015 0.022 -0.6 0.044 0.004 0.015 0.022 -0.4 0.043 0.003 0.015 0.022 -0.2 0.044 0.004 0.015 0.022 0 0.044 0.004 0.015 0.021 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.022 0.004 0.018 0.023 -0.6 0.022 0.004 0.018 0.023 -0.4 0.022 0.004 0.019 0.023 -0.2 0.022 0.004 0.019 0.023 0 0.022 0.004 0.019 0.023 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.002 0.003 0.016 0.013 -0.6 0.002 0.003 0.016 0.013 -0.4 0.002 0.003 0.015 0.013 -0.2 0.002 0.003 0.015 0.012 0 0.002 0.003 0.015 0.012 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.031 0.002 0.011 0.017 -0.6 0.031 0.002 0.012 0.017 -0.4 0.031 0.002 0.012 0.017 -0.2 0.031 0.002 0.013 0.017 0 0.031 0.002 0.012 0.017 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.016 0.003 0.016 0.017 -0.6 0.016 0.003 0.015 0.016 -0.4 0.015 0.003 0.016 0.017 -0.2 0.015 0.003 0.013 0.017 0 0.016 0.003 0.014 0.017 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.002 0.002 0.011 0.010 -0.6 0.002 0.002 0.014 0.010 -0.4 0.001 0.002 0.013 0.010 -0.2 0.001 0.002 0.011 0.010 0 0.001 0.002 0.011 0.009 > > ########################## Sample mean true power ######################### > TPframe = aggregate(SampleMeanPower,by=list(Correlation, Method, PopMeanPower,Ntests),FUN=mean, na.rm=T) > tp = TPframe$x; dim(tp) = c(5,4,3,5) > TPOW = kounts > for(j in 1:5) TPOW[,,,j] = tp[,,,j] > cat("\n Sample mean true power \n\n") Sample mean true power > print(round(TPOW,3)) , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.252 0.252 0.252 0.252 -0.6 0.251 0.251 0.251 0.251 -0.4 0.250 0.250 0.250 0.250 -0.2 0.250 0.250 0.250 0.250 0 0.250 0.250 0.250 0.250 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.497 0.497 0.497 0.497 -0.6 0.499 0.499 0.499 0.499 -0.4 0.499 0.499 0.499 0.499 -0.2 0.499 0.499 0.499 0.499 0 0.500 0.500 0.500 0.500 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.739 0.739 0.739 0.739 -0.6 0.744 0.744 0.744 0.744 -0.4 0.745 0.745 0.745 0.745 -0.2 0.749 0.749 0.749 0.749 0 0.750 0.750 0.750 0.750 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.252 0.252 0.252 0.252 -0.6 0.251 0.251 0.251 0.251 -0.4 0.250 0.250 0.250 0.250 -0.2 0.250 0.250 0.250 0.250 0 0.250 0.250 0.250 0.250 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.498 0.498 0.498 0.498 -0.6 0.498 0.498 0.498 0.498 -0.4 0.499 0.499 0.499 0.499 -0.2 0.500 0.500 0.500 0.500 0 0.500 0.500 0.500 0.500 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.739 0.739 0.739 0.739 -0.6 0.744 0.744 0.744 0.744 -0.4 0.746 0.746 0.746 0.746 -0.2 0.748 0.748 0.748 0.748 0 0.750 0.750 0.750 0.750 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.252 0.252 0.252 0.252 -0.6 0.251 0.251 0.251 0.251 -0.4 0.250 0.250 0.250 0.250 -0.2 0.250 0.250 0.250 0.250 0 0.250 0.250 0.250 0.250 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.498 0.498 0.498 0.498 -0.6 0.499 0.499 0.499 0.499 -0.4 0.499 0.499 0.499 0.499 -0.2 0.500 0.500 0.500 0.500 0 0.500 0.500 0.500 0.500 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.739 0.739 0.739 0.739 -0.6 0.744 0.744 0.744 0.744 -0.4 0.746 0.746 0.746 0.746 -0.2 0.748 0.748 0.748 0.748 0 0.750 0.750 0.750 0.750 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.252 0.252 0.252 0.252 -0.6 0.251 0.251 0.251 0.251 -0.4 0.250 0.250 0.250 0.250 -0.2 0.250 0.250 0.250 0.250 0 0.250 0.250 0.250 0.250 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.497 0.497 0.497 0.497 -0.6 0.499 0.499 0.499 0.499 -0.4 0.499 0.499 0.499 0.499 -0.2 0.500 0.500 0.500 0.500 0 0.500 0.500 0.500 0.500 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.739 0.739 0.739 0.739 -0.6 0.744 0.744 0.744 0.744 -0.4 0.746 0.746 0.746 0.746 -0.2 0.748 0.748 0.748 0.748 0 0.750 0.750 0.750 0.750 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.252 0.252 0.252 0.252 -0.6 0.251 0.251 0.251 0.251 -0.4 0.251 0.251 0.251 0.251 -0.2 0.250 0.250 0.250 0.250 0 0.250 0.250 0.250 0.250 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.498 0.498 0.498 0.498 -0.6 0.499 0.499 0.499 0.499 -0.4 0.499 0.499 0.499 0.499 -0.2 0.500 0.500 0.500 0.500 0 0.500 0.500 0.500 0.500 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.739 0.739 0.739 0.739 -0.6 0.744 0.744 0.744 0.744 -0.4 0.746 0.746 0.746 0.746 -0.2 0.748 0.748 0.748 0.748 0 0.750 0.750 0.750 0.750 > > > ###################### Bias ###################### > biasframe = aggregate(Bias,by=list(Correlation, Method, PopMeanPower,Ntests),FUN=mean, na.rm=T) > meanbias = biasframe$x; dim(meanbias) = c(5,4,3,5) > MeanBias = kounts > for(j in 1:5) MeanBias[,,,j] = meanbias[,,,j] > cat("\n Mean Bias \n\n") Mean Bias > print(round(MeanBias,3)) , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.123 0.603 0.051 -0.003 -0.6 0.119 0.602 0.049 -0.005 -0.4 0.116 0.602 0.049 -0.005 -0.2 0.118 0.602 0.049 -0.005 0 0.117 0.602 0.048 -0.006 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.321 0.406 0.033 0.032 -0.6 0.323 0.406 0.035 0.034 -0.4 0.322 0.406 0.035 0.034 -0.2 0.322 0.406 0.035 0.035 0 0.324 0.407 0.037 0.037 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.238 0.214 0.072 0.070 -0.6 0.239 0.215 0.078 0.074 -0.4 0.239 0.216 0.079 0.075 -0.2 0.240 0.216 0.083 0.077 0 0.240 0.217 0.085 0.079 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.141 0.603 0.051 -0.013 -0.6 0.139 0.602 0.050 -0.014 -0.4 0.137 0.602 0.050 -0.014 -0.2 0.137 0.602 0.049 -0.015 0 0.137 0.602 0.049 -0.015 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.332 0.406 0.033 0.012 -0.6 0.333 0.406 0.034 0.014 -0.4 0.334 0.406 0.034 0.014 -0.2 0.334 0.406 0.035 0.015 0 0.335 0.406 0.034 0.015 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.240 0.214 0.075 0.058 -0.6 0.241 0.216 0.081 0.062 -0.4 0.241 0.216 0.083 0.064 -0.2 0.241 0.217 0.086 0.066 0 0.242 0.217 0.088 0.067 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.151 0.603 0.051 -0.016 -0.6 0.148 0.602 0.050 -0.017 -0.4 0.147 0.602 0.050 -0.018 -0.2 0.147 0.602 0.050 -0.018 0 0.146 0.602 0.050 -0.018 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.336 0.406 0.032 0.001 -0.6 0.338 0.406 0.033 0.002 -0.4 0.338 0.406 0.033 0.002 -0.2 0.339 0.406 0.034 0.003 0 0.339 0.406 0.035 0.004 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.240 0.214 0.075 0.046 -0.6 0.241 0.216 0.082 0.051 -0.4 0.242 0.216 0.085 0.054 -0.2 0.242 0.217 0.087 0.055 0 0.242 0.217 0.089 0.057 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.157 0.603 0.052 -0.018 -0.6 0.155 0.602 0.051 -0.019 -0.4 0.153 0.602 0.050 -0.020 -0.2 0.153 0.602 0.050 -0.019 0 0.152 0.602 0.050 -0.020 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.339 0.406 0.032 -0.007 -0.6 0.340 0.406 0.033 -0.006 -0.4 0.341 0.406 0.033 -0.005 -0.2 0.341 0.406 0.034 -0.005 0 0.341 0.406 0.034 -0.005 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.240 0.214 0.076 0.035 -0.6 0.241 0.216 0.082 0.040 -0.4 0.242 0.216 0.086 0.043 -0.2 0.242 0.217 0.088 0.044 0 0.242 0.217 0.090 0.046 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.161 0.603 0.052 -0.018 -0.6 0.159 0.602 0.052 -0.019 -0.4 0.158 0.602 0.051 -0.020 -0.2 0.157 0.602 0.051 -0.020 0 0.157 0.602 0.051 -0.020 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.340 0.406 0.032 -0.013 -0.6 0.341 0.406 0.033 -0.012 -0.4 0.342 0.406 0.033 -0.011 -0.2 0.342 0.406 0.034 -0.011 0 0.342 0.406 0.034 -0.010 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 0.241 0.214 0.077 0.024 -0.6 0.241 0.216 0.083 0.029 -0.4 0.242 0.216 0.086 0.031 -0.2 0.242 0.217 0.088 0.033 0 0.243 0.217 0.090 0.035 > > ########################## Z-tests for bias ######################### > # Calculate Z > ZforBias = kounts > for(j in 1:5) ZforBias[,,,j] = + sqrt(kounts[1,1,1,1])*(MeanEstimatedPower[,,,j] - TPOW[,,,j])/SDestimatedPower[,,,j] > cat("\n Z-tests for bias \n", + " Any Z with |Z|>4.61 is significant.\n\n") Z-tests for bias Any Z with |Z|>4.61 is significant. > print(round(ZforBias,2)) , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 92.15 5465.18 112.93 -9.41 -0.6 90.27 5451.12 113.08 -10.44 -0.4 88.63 5455.77 113.59 -9.31 -0.2 89.60 5428.67 113.58 -8.81 0 90.18 5498.71 112.40 -11.11 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 422.82 3240.33 63.25 46.54 -0.6 437.73 3271.93 63.43 47.22 -0.4 436.40 3295.04 64.09 47.81 -0.2 432.90 3263.03 64.09 47.68 0 441.18 3254.25 64.28 49.60 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 2665.59 2315.00 160.57 240.93 -0.6 2769.44 2281.70 166.45 243.48 -0.4 2788.30 2293.89 168.09 248.81 -0.2 2954.76 2298.66 172.01 251.05 0 3026.34 2286.81 173.04 253.34 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 160.96 8562.41 179.12 -40.21 -0.6 159.99 8610.23 174.89 -39.32 -0.4 157.80 8548.14 177.92 -39.38 -0.2 160.57 8668.13 179.48 -40.60 0 159.10 8602.50 177.59 -39.33 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 732.53 5127.24 96.29 30.64 -0.6 737.81 5130.77 98.36 33.20 -0.4 734.37 5111.86 98.36 32.51 -0.2 727.72 5029.48 96.24 32.20 0 739.95 5098.78 94.03 30.70 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 4857.54 3626.53 264.78 298.97 -0.6 5172.49 3623.34 277.51 308.49 -0.4 5296.67 3631.00 284.29 315.00 -0.2 5429.48 3584.17 286.22 318.88 0 5610.23 3644.32 293.73 324.30 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 243.25 12165.81 241.78 -65.42 -0.6 239.26 12157.28 248.71 -64.42 -0.4 240.05 12218.88 236.58 -65.43 -0.2 234.07 11943.69 234.97 -64.93 0 235.41 12067.10 250.06 -65.42 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 1064.78 7224.83 124.26 9.65 -0.6 1076.32 7238.36 132.93 11.02 -0.4 1079.42 7184.70 128.39 9.32 -0.2 1091.21 7189.40 133.66 11.46 0 1082.46 7219.77 136.18 11.71 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 7267.24 5134.77 380.91 330.54 -0.6 7682.56 5060.64 396.39 340.70 -0.4 8001.43 5103.07 408.40 352.70 -0.2 8055.55 5078.37 415.78 354.08 0 8536.79 5071.21 421.17 357.67 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 359.63 17337.22 331.43 -90.81 -0.6 351.08 17135.83 332.65 -91.84 -0.4 353.03 17361.66 331.99 -92.37 -0.2 346.10 17051.94 330.85 -89.87 0 348.49 17161.76 332.37 -93.49 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 1549.69 10290.58 192.04 -19.77 -0.6 1546.92 10212.22 185.62 -20.61 -0.4 1558.51 10222.75 176.28 -20.77 -0.2 1551.02 10203.52 182.52 -19.68 0 1555.94 10175.66 179.16 -19.61 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 10549.75 7249.19 547.95 347.00 -0.6 11222.34 7189.86 570.45 360.74 -0.4 11613.36 7207.89 583.84 371.29 -0.2 12089.28 7222.44 595.37 375.52 0 12378.25 7162.26 588.19 375.89 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 514.42 24198.56 441.45 -115.78 -0.6 511.42 24387.83 437.53 -114.32 -0.4 509.46 24402.60 439.46 -116.74 -0.2 501.71 24166.06 401.96 -115.13 0 509.72 24559.59 432.56 -117.69 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 2194.53 14416.46 214.69 -63.30 -0.6 2212.47 14417.85 235.22 -61.07 -0.4 2218.67 14385.07 218.80 -61.55 -0.2 2225.88 14472.45 266.24 -61.66 0 2197.45 14150.77 238.41 -61.96 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 14891.62 10173.25 770.28 341.48 -0.6 16225.60 10172.69 639.70 361.84 -0.4 16803.14 10192.28 668.07 366.36 -0.2 17229.94 10142.72 832.33 369.12 0 17946.34 10236.69 845.24 377.46 > > # Protect the 300 tests in this table with Bonferroni > # Two-tailed at the 0.001 significance level > # 0.0005/300 = 0.000001666667, and qnorm(0.0005/300) = -4.649133 > # So any Z with |Z|>4.65 is significant. > > > ###################### Is estimate close enough? ############# > > # Get table of counts in a different order, for labels > k2 = table(Method, Ntests, PopMeanPower, Correlation) > > closeframe = + aggregate(CloseEnough,by=list(Method, Ntests, PopMeanPower, Correlation),FUN=mean, na.rm=T) > klose = closeframe$x; dim(klose) = c(4,5,3,5) > ProportionWithin.05 = k2 > for(j in 1:5) ProportionWithin.05[,,,j] = klose[,,,j] > cat("\n Proportion of estimates within 0.05 of target \n\n") Proportion of estimates within 0.05 of target > print(round(ProportionWithin.05,3)) , , PopMeanPower = 0.25, Correlation = -0.8 Ntests Method 100 250 500 1000 2000 Pcurve 0.221 0.138 0.051 0.006 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.493 0.494 0.476 0.451 0.420 Zcurve 0.610 0.794 0.883 0.942 0.983 , , PopMeanPower = 0.5, Correlation = -0.8 Ntests Method 100 250 500 1000 2000 Pcurve 0.003 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.551 0.686 0.762 0.845 0.919 Zcurve 0.448 0.699 0.872 0.963 0.989 , , PopMeanPower = 0.75, Correlation = -0.8 Ntests Method 100 250 500 1000 2000 Pcurve 0.000 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.272 0.204 0.130 0.056 0.015 Zcurve 0.224 0.329 0.552 0.876 0.997 , , PopMeanPower = 0.25, Correlation = -0.6 Ntests Method 100 250 500 1000 2000 Pcurve 0.221 0.147 0.056 0.007 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.512 0.504 0.499 0.478 0.443 Zcurve 0.617 0.796 0.878 0.942 0.979 , , PopMeanPower = 0.5, Correlation = -0.6 Ntests Method 100 250 500 1000 2000 Pcurve 0.003 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.549 0.669 0.745 0.827 0.907 Zcurve 0.449 0.699 0.874 0.966 0.992 , , PopMeanPower = 0.75, Correlation = -0.6 Ntests Method 100 250 500 1000 2000 Pcurve 0.000 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.234 0.159 0.084 0.025 0.002 Zcurve 0.189 0.261 0.445 0.777 0.986 , , PopMeanPower = 0.25, Correlation = -0.4 Ntests Method 100 250 500 1000 2000 Pcurve 0.222 0.152 0.055 0.006 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.510 0.512 0.496 0.490 0.453 Zcurve 0.614 0.785 0.874 0.939 0.980 , , PopMeanPower = 0.5, Correlation = -0.4 Ntests Method 100 250 500 1000 2000 Pcurve 0.003 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.551 0.669 0.747 0.813 0.898 Zcurve 0.445 0.697 0.875 0.966 0.992 , , PopMeanPower = 0.75, Correlation = -0.4 Ntests Method 100 250 500 1000 2000 Pcurve 0.000 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.230 0.137 0.062 0.015 0.001 Zcurve 0.183 0.235 0.378 0.711 0.977 , , PopMeanPower = 0.25, Correlation = -0.2 Ntests Method 100 250 500 1000 2000 Pcurve 0.215 0.148 0.060 0.010 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.508 0.510 0.505 0.479 0.468 Zcurve 0.616 0.790 0.872 0.937 0.979 , , PopMeanPower = 0.5, Correlation = -0.2 Ntests Method 100 250 500 1000 2000 Pcurve 0.003 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.546 0.656 0.735 0.812 0.900 Zcurve 0.448 0.682 0.872 0.966 0.992 , , PopMeanPower = 0.75, Correlation = -0.2 Ntests Method 100 250 500 1000 2000 Pcurve 0.000 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.210 0.122 0.051 0.010 0.001 Zcurve 0.162 0.208 0.339 0.664 0.961 , , PopMeanPower = 0.25, Correlation = 0 Ntests Method 100 250 500 1000 2000 Pcurve 0.218 0.147 0.062 0.008 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.510 0.516 0.503 0.504 0.473 Zcurve 0.614 0.787 0.876 0.935 0.979 , , PopMeanPower = 0.5, Correlation = 0 Ntests Method 100 250 500 1000 2000 Pcurve 0.002 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.546 0.662 0.730 0.812 0.888 Zcurve 0.449 0.692 0.871 0.965 0.993 , , PopMeanPower = 0.75, Correlation = 0 Ntests Method 100 250 500 1000 2000 Pcurve 0.000 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.200 0.113 0.044 0.009 0.000 Zcurve 0.154 0.187 0.309 0.613 0.951 > ###################### Mean Absolute Error ###################### > # Take a look at marginals to see roughly who won overall > cat(" Marginal means for method \n") Marginal means for method > print(aggregate(round(100*AbsError,2),by=list(Method),FUN=mean, na.rm=T)) Group.1 x 1 Pcurve 24.171714 2 Puniform 40.817032 3 MaxLike 5.825573 4 Zcurve 3.883568 > > errframe = aggregate(AbsError,by=list(Correlation, Method, PopMeanPower,Ntests),FUN=mean, na.rm=T) > meanerr = 100*errframe$x; dim(meanerr) = c(5,4,3,5) > MeanAbsoluteError = kounts > for(j in 1:5) MeanAbsoluteError[,,,j] = meanerr[,,,j] > cat("\n Mean Absolute Error of Prediction \n\n") Mean Absolute Error of Prediction > print(round(MeanAbsoluteError,2)) , , PopMeanPower = 0.25, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 14.51 60.28 5.52 4.58 -0.6 14.27 60.24 5.39 4.57 -0.4 14.10 60.23 5.39 4.59 -0.2 14.35 60.23 5.38 4.57 0 14.15 60.23 5.38 4.55 , , PopMeanPower = 0.5, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 32.07 40.60 5.26 6.60 -0.6 32.30 40.62 5.26 6.58 -0.4 32.25 40.62 5.25 6.59 -0.2 32.22 40.63 5.29 6.63 0 32.44 40.67 5.37 6.66 , , PopMeanPower = 0.75, Ntests = 100 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 23.81 21.42 7.87 7.19 -0.6 23.91 21.54 8.35 7.53 -0.4 23.94 21.56 8.43 7.63 -0.2 24.01 21.64 8.71 7.84 0 24.04 21.67 8.86 7.95 , , PopMeanPower = 0.25, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 14.44 60.27 5.16 3.14 -0.6 14.27 60.25 5.09 3.17 -0.4 14.07 60.23 5.04 3.21 -0.2 14.01 60.23 5.02 3.21 0 14.10 60.23 5.02 3.23 , , PopMeanPower = 0.5, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 33.20 40.60 3.94 3.87 -0.6 33.32 40.61 4.01 3.88 -0.4 33.38 40.63 4.04 3.89 -0.2 33.42 40.64 4.10 3.97 0 33.46 40.64 4.07 3.91 , , PopMeanPower = 0.75, Ntests = 250 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 23.96 21.43 7.54 5.81 -0.6 24.06 21.55 8.14 6.22 -0.4 24.10 21.60 8.36 6.38 -0.2 24.14 21.66 8.66 6.61 0 24.17 21.70 8.83 6.72 , , PopMeanPower = 0.25, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 15.10 60.26 5.15 2.64 -0.6 14.78 60.23 5.04 2.71 -0.4 14.73 60.23 5.03 2.70 -0.2 14.73 60.23 5.03 2.71 0 14.66 60.22 5.00 2.70 , , PopMeanPower = 0.5, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 33.62 40.59 3.48 2.61 -0.6 33.78 40.62 3.57 2.62 -0.4 33.77 40.62 3.57 2.61 -0.2 33.90 40.64 3.63 2.62 0 33.89 40.64 3.65 2.63 , , PopMeanPower = 0.75, Ntests = 500 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 24.01 21.43 7.55 4.66 -0.6 24.11 21.55 8.16 5.12 -0.4 24.16 21.63 8.52 5.40 -0.2 24.19 21.67 8.71 5.53 0 24.22 21.71 8.92 5.69 , , PopMeanPower = 0.25, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 15.67 60.26 5.17 2.37 -0.6 15.49 60.24 5.11 2.41 -0.4 15.33 60.23 5.05 2.47 -0.2 15.30 60.22 5.05 2.48 0 15.24 60.22 5.01 2.50 , , PopMeanPower = 0.5, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 33.88 40.59 3.25 1.92 -0.6 33.99 40.61 3.34 1.91 -0.4 34.07 40.63 3.42 1.89 -0.2 34.09 40.63 3.43 1.90 0 34.11 40.64 3.46 1.89 , , PopMeanPower = 0.75, Ntests = 1000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 24.04 21.43 7.62 3.51 -0.6 24.13 21.56 8.23 4.01 -0.4 24.18 21.63 8.56 4.27 -0.2 24.21 21.67 8.76 4.43 0 24.24 21.72 8.97 4.59 , , PopMeanPower = 0.25, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 16.09 60.26 5.23 2.14 -0.6 15.85 60.24 5.16 2.22 -0.4 15.81 60.23 5.14 2.24 -0.2 15.70 60.22 5.10 2.26 0 15.71 60.22 5.09 2.27 , , PopMeanPower = 0.5, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 33.99 40.59 3.20 1.72 -0.6 34.11 40.61 3.30 1.63 -0.4 34.19 40.63 3.37 1.61 -0.2 34.23 40.63 3.38 1.58 0 34.25 40.64 3.42 1.59 , , PopMeanPower = 0.75, Ntests = 2000 Method Correlation Pcurve Puniform MaxLike Zcurve -0.8 24.05 21.44 7.66 2.40 -0.6 24.15 21.56 8.28 2.90 -0.4 24.19 21.63 8.57 3.14 -0.2 24.22 21.68 8.80 3.34 0 24.25 21.72 9.00 3.50 > > > ################################################################################## > # Need to do all 6 pairwise matched t-tests for mean absolute error, for each > # combination of Ntests, PopMeanPower and Correlation. 5*3*5*6 = 450 > # with a Bonferroni correction for tests in the table. > # alpha = 0.001; a = alpha/450; critZ = qnorm(1-a/2); critZ # 4.732088 > # Return to simdata, multivariate format. > ###################################################################################### > > # Bonferroni correct at joint 0.001 level for 6 pairwise comparisons within > # each combination of > # 5 values of Corr(N,ES) after selection > # 5 values of k = Number of studies > # 3 values of mean power after selection > # 6*75 = 450 > a = 0.001/450; critZ = qnorm(1-a/2) # 4.732088 > > ae = simdata[,6:9] # The estimates > corr = simdata[,2]; kk = simdata[,3]; pow = simdata[,4]; Index = 1:nlines > for(kol in 1:4) ae[,kol] = abs(ae[,kol]-pow) # Now they are absolute errors. > > nstudies = unique(Ntests); truepower = unique(PopMeanPower) > correlation = unique(Correlation) > pairwise = matrix(0,4,4); rownames(pairwise) = colnames(pairwise) = Methods > keepscore = pairwise # Count winners in this matrix > > # This does not have to be efficient, and it isn't. > > cat("\n\n Matched Z-tests for pairwise differences in mean absolute error. \n", + "Positive value means the row method is less accurate on average. \n", + "Critical value Bonferroni protected at the 0.001 level is",critZ,".\n\n") Matched Z-tests for pairwise differences in mean absolute error. Positive value means the row method is less accurate on average. Critical value Bonferroni protected at the 0.001 level is 4.732088 . > > for(h in 1:5) # Looping over correlation between effect size and sample size + { + for(i in 1:5) # Looping over Ntests + { + for(j in 1:3) # Looping over PopMeanPower + { + cat("\n Correlation =",correlation[h]," Ntests =",nstudies[i]," and PopMeanPower =",truepower[j],"\n\n") + In = (corr==correlation[h]) * (kk==nstudies[i]) * (pow==truepower[j]) + pick = Index[In==1] + # Fill matrix of pairwise tests in a double loop + for(K in 1:4) + { + for(L in 1:4) + { + differ = (ae[,K]-ae[,L])[pick] + Z = 0 + if(K != L) Z = t.test(differ)$statistic + if(K < L) pairwise[K,L] = Z + if(Z < -critZ) keepscore[K,L] = keepscore[K,L]+1 + } # Next L + } # Next K + print(round(pairwise[1:3,2:4],2)) + } # Next j (PopMeanPower) + } # Next i (Ntests) + } # Next h (Correlation) Correlation = 0 Ntests = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -487.79 114.35 91.75 Puniform 0.00 1981.09 1526.99 MaxLike 0.00 0.00 18.09 Correlation = 0 Ntests = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -132.47 432.48 371.35 Puniform 0.00 1045.77 781.17 MaxLike 0.00 0.00 -58.60 Correlation = 0 Ntests = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 445.43 415.19 715.55 Puniform 0.00 387.21 695.29 MaxLike 0.00 0.00 48.66 Correlation = 0 Ntests = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -624.77 149.15 120.16 Puniform 0.00 2695.67 2134.78 MaxLike 0.00 0.00 44.50 Correlation = 0 Ntests = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -189.05 883.4 621.98 Puniform 0.00 1585.4 1342.05 MaxLike 0.00 0.0 6.99 Correlation = 0 Ntests = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 768.96 593.79 1047.35 Puniform 0.00 547.64 1029.53 MaxLike 0.00 0.00 190.87 Correlation = 0 Ntests = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -802.07 204.40 165.05 Puniform 0.00 3530.91 2643.95 MaxLike 0.00 0.00 68.89 Correlation = 0 Ntests = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -257.55 1595.49 871.71 Puniform 0.00 2115.92 1942.18 MaxLike 0.00 0.00 41.50 Correlation = 0 Ntests = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1080.53 812.13 1373.44 Puniform 0.00 751.50 1362.75 MaxLike 0.00 0.00 437.51 Correlation = 0 Ntests = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1110.7 299.29 240.47 Puniform 0.0 4601.68 3256.33 MaxLike 0.0 0.00 94.65 Correlation = 0 Ntests = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -356.25 2427.88 1115.56 Puniform 0.00 2589.81 2499.99 MaxLike 0.00 0.00 63.80 Correlation = 0 Ntests = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1528.69 1141.19 1844.31 Puniform 0.00 1055.27 1827.97 MaxLike 0.00 0.00 761.72 Correlation = 0 Ntests = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1558.4 429.44 353.71 Puniform 0.0 5701.91 3966.34 MaxLike 0.0 0.00 132.96 Correlation = 0 Ntests = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -491.95 3314.19 1385.97 Puniform 0.00 3306.34 2982.65 MaxLike 0.00 0.00 83.95 Correlation = 0 Ntests = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 2168.83 1605.57 2504.51 Puniform 0.00 1482.91 2471.09 MaxLike 0.00 0.00 1139.54 Correlation = -0.2 Ntests = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -480.8 115.84 93.36 Puniform 0.0 1980.62 1533.85 MaxLike 0.0 0.00 17.80 Correlation = -0.2 Ntests = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -132.76 417.73 353.57 Puniform 0.00 1079.41 782.47 MaxLike 0.00 0.00 -61.47 Correlation = -0.2 Ntests = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 445.71 422.14 720.74 Puniform 0.00 393.47 696.03 MaxLike 0.00 0.00 47.01 Correlation = -0.2 Ntests = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -632.74 149.67 120.41 Puniform 0.00 2742.20 2158.34 MaxLike 0.00 0.00 45.30 Correlation = -0.2 Ntests = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -187.02 872.69 613.87 Puniform 0.00 1603.37 1331.53 MaxLike 0.00 0.00 5.71 Correlation = -0.2 Ntests = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 755.86 586.95 1033.26 Puniform 0.00 541.23 1013.71 MaxLike 0.00 0.00 181.21 Correlation = -0.2 Ntests = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -794.41 201.82 165.15 Puniform 0.00 3297.64 2627.09 MaxLike 0.00 0.00 67.98 Correlation = -0.2 Ntests = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -259.66 1589.10 876.35 Puniform 0.00 2053.05 1925.58 MaxLike 0.00 0.00 40.17 Correlation = -0.2 Ntests = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1107.65 821.21 1382.35 Puniform 0.00 755.53 1358.86 MaxLike 0.00 0.00 426.45 Correlation = -0.2 Ntests = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1100.87 297.92 240.24 Puniform 0.00 4521.30 3270.53 MaxLike 0.00 0.00 96.29 Correlation = -0.2 Ntests = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -355.78 2435.33 1103.32 Puniform 0.00 2608.44 2474.01 MaxLike 0.00 0.00 61.98 Correlation = -0.2 Ntests = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1554.06 1156.11 1856.50 Puniform 0.00 1067.27 1833.08 MaxLike 0.00 0.00 776.12 Correlation = -0.2 Ntests = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1537.81 423.26 348.63 Puniform 0.00 5470.94 3944.38 MaxLike 0.00 0.00 132.31 Correlation = -0.2 Ntests = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -498.35 3748.15 1374.27 Puniform 0.00 3639.64 2921.29 MaxLike 0.00 0.00 83.40 Correlation = -0.2 Ntests = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 2187.04 1600.68 2462.99 Puniform 0.00 1473.98 2419.90 MaxLike 0.00 0.00 1138.87 Correlation = -0.4 Ntests = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -484.03 112.70 91.45 Puniform 0.00 1979.01 1515.97 MaxLike 0.00 0.00 17.52 Correlation = -0.4 Ntests = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -133.83 416.30 353.46 Puniform 0.00 1073.91 779.56 MaxLike 0.00 0.00 -60.50 Correlation = -0.4 Ntests = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 440.41 430.00 728.11 Puniform 0.00 399.54 695.94 MaxLike 0.00 0.00 42.58 Correlation = -0.4 Ntests = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -616.04 145.90 118.88 Puniform 0.00 2707.09 2160.93 MaxLike 0.00 0.00 45.82 Correlation = -0.4 Ntests = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -189.19 887.23 616.83 Puniform 0.00 1624.84 1327.27 MaxLike 0.00 0.00 6.58 Correlation = -0.4 Ntests = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 772.97 598.30 1032.05 Puniform 0.00 549.12 1000.34 MaxLike 0.00 0.00 178.03 Correlation = -0.4 Ntests = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -814.56 206.23 169.05 Puniform 0.00 3313.63 2657.72 MaxLike 0.00 0.00 69.11 Correlation = -0.4 Ntests = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -261.27 1498.41 849.70 Puniform 0.00 2009.28 1909.43 MaxLike 0.00 0.00 36.60 Correlation = -0.4 Ntests = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1113.83 818.12 1377.35 Puniform 0.00 751.25 1349.71 MaxLike 0.00 0.00 413.12 Correlation = -0.4 Ntests = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1119.56 304.75 244.78 Puniform 0.00 4611.59 3295.93 MaxLike 0.00 0.00 97.93 Correlation = -0.4 Ntests = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -358.21 2312.32 1103.33 Puniform 0.00 2546.99 2480.69 MaxLike 0.00 0.00 61.08 Correlation = -0.4 Ntests = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1589.85 1157.82 1844.36 Puniform 0.00 1063.50 1807.05 MaxLike 0.00 0.00 757.71 Correlation = -0.4 Ntests = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1551.07 434.21 353.35 Puniform 0.00 5725.05 3939.14 MaxLike 0.00 0.00 136.35 Correlation = -0.4 Ntests = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -499.56 2844.87 1369.19 Puniform 0.00 2990.09 2911.05 MaxLike 0.00 0.00 77.39 Correlation = -0.4 Ntests = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 2227.18 1365.66 2482.27 Puniform 0.00 1219.10 2430.50 MaxLike 0.00 0.00 704.42 Correlation = -0.6 Ntests = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -479.27 114.61 92.69 Puniform 0.00 1963.00 1532.25 MaxLike 0.00 0.00 17.78 Correlation = -0.6 Ntests = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -133.54 420.67 356.96 Puniform 0.00 1070.93 779.62 MaxLike 0.00 0.00 -60.50 Correlation = -0.6 Ntests = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 436.52 432.89 722.62 Puniform 0.00 402.90 693.08 MaxLike 0.00 0.00 43.74 Correlation = -0.6 Ntests = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -614.18 148.46 121.90 Puniform 0.00 2599.76 2174.89 MaxLike 0.00 0.00 47.95 Correlation = -0.6 Ntests = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -191.2 873.17 615.45 Puniform 0.0 1634.11 1337.10 MaxLike 0.0 0.00 5.76 Correlation = -0.6 Ntests = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 793.87 603.32 1028.15 Puniform 0.00 552.23 990.27 MaxLike 0.00 0.00 173.19 Correlation = -0.6 Ntests = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -806.51 208.49 168.54 Puniform 0.00 3560.07 2686.77 MaxLike 0.00 0.00 70.45 Correlation = -0.6 Ntests = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -259.09 1500.52 848.41 Puniform 0.00 2077.50 1920.28 MaxLike 0.00 0.00 37.36 Correlation = -0.6 Ntests = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1136.27 820.82 1364.63 Puniform 0.00 751.49 1326.42 MaxLike 0.00 0.00 398.04 Correlation = -0.6 Ntests = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1102.16 302.69 247.27 Puniform 0.00 4557.22 3318.24 MaxLike 0.00 0.00 102.73 Correlation = -0.6 Ntests = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -359.32 2472.8 1085.83 Puniform 0.00 2690.9 2453.47 MaxLike 0.00 0.0 57.60 Correlation = -0.6 Ntests = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1599.14 1157.72 1821.37 Puniform 0.00 1061.50 1773.95 MaxLike 0.00 0.00 760.02 Correlation = -0.6 Ntests = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1554.1 433.54 357.33 Puniform 0.0 5639.52 3980.05 MaxLike 0.0 0.00 139.24 Correlation = -0.6 Ntests = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -502.66 3070.63 1350.81 Puniform 0.00 3194.27 2872.96 MaxLike 0.00 0.00 74.14 Correlation = -0.6 Ntests = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 2282.96 1357.34 2475.56 Puniform 0.00 1207.29 2412.13 MaxLike 0.00 0.00 672.47 Correlation = -0.8 Ntests = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -474.02 115.23 94.09 Puniform 0.00 1958.59 1536.58 MaxLike 0.00 0.00 20.80 Correlation = -0.8 Ntests = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -132.18 395.97 337.15 Puniform 0.00 1060.18 773.28 MaxLike 0.00 0.00 -61.01 Correlation = -0.8 Ntests = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 440.34 449.65 732.79 Puniform 0.00 417.50 692.19 MaxLike 0.00 0.00 35.85 Correlation = -0.8 Ntests = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -614.1 151.22 124.93 Puniform 0.0 2737.72 2180.45 MaxLike 0.0 0.00 51.28 Correlation = -0.8 Ntests = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -192.61 869.95 604.17 Puniform 0.00 1626.75 1334.47 MaxLike 0.00 0.00 2.90 Correlation = -0.8 Ntests = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 793.11 615.42 1038.48 Puniform 0.00 561.00 987.80 MaxLike 0.00 0.00 154.28 Correlation = -0.8 Ntests = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -805.44 213.19 175.45 Puniform 0.00 3516.03 2704.83 MaxLike 0.00 0.00 76.66 Correlation = -0.8 Ntests = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -261.54 1360.39 815.40 Puniform 0.00 1916.91 1861.07 MaxLike 0.00 0.00 32.08 Correlation = -0.8 Ntests = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1152.16 836.6 1374.83 Puniform 0.00 761.6 1321.73 MaxLike 0.00 0.0 368.00 Correlation = -0.8 Ntests = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1117.3 313.16 254.39 Puniform 0.0 4562.39 3316.48 MaxLike 0.0 0.00 106.44 Correlation = -0.8 Ntests = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -363.96 2495.46 1058.89 Puniform 0.00 2794.47 2381.90 MaxLike 0.00 0.00 52.67 Correlation = -0.8 Ntests = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 1656.99 1175.73 1836.85 Puniform 0.00 1069.99 1770.47 MaxLike 0.00 0.00 716.29 Correlation = -0.8 Ntests = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1542.37 441.17 364.68 Puniform 0.00 5766.13 4004.38 MaxLike 0.00 0.00 147.15 Correlation = -0.8 Ntests = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -506.51 2728.95 1312.46 Puniform 0.00 2931.74 2784.98 MaxLike 0.00 0.00 62.72 Correlation = -0.8 Ntests = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve 2366.42 1632.41 2495.18 Puniform 0.00 1481.70 2401.75 MaxLike 0.00 0.00 1076.47 > > cat("\nTotal number of significant comparisons: \n") Total number of significant comparisons: > print(sum(keepscore)) [1] 449 > > cat("\n\n Number of times row method is significantly more accurate than column method \n\n"); print(addmargins(keepscore,2)) Number of times row method is significantly more accurate than column method Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 50 0 0 50 Puniform 25 0 0 0 25 MaxLike 75 75 0 5 155 Zcurve 75 75 69 0 219 > > > > > # Now do it separately by k = nstudies. This is crude. > > for(i in 1:5) # Looping over Ntests + { + keepscore = matrix(0,4,4) # Start fresh for each value of Ntests + rownames(keepscore) = colnames(keepscore) = Methods + for(h in 1:5) # Looping over correlation between effect size and sample size + { + for(j in 1:3) # Looping over PopMeanPower + { + # cat("\n Correlation =",correlation[h]," Ntests =",nstudies[i]," and PopMeanPower =",truepower[j],"\n\n") + In = (corr==correlation[h]) * (kk==nstudies[i]) * (pow==truepower[j]) + pick = Index[In==1] + # Fill matrix of pairwise tests in a double loop + for(K in 1:4) + { + for(L in 1:4) + { + differ = (ae[,K]-ae[,L])[pick] + Z = 0 + if(K != L) Z = t.test(differ)$statistic + if(K < L) pairwise[K,L] = Z + if(Z < -critZ) keepscore[K,L] = keepscore[K,L]+1 + } # Next L + } # Next K + } # Next j (PopMeanPower) + } # Next h (Correlation) + # Now output for this value of Ntests + cat("\n\n Number of times row method is significantly more accurate than column method for",nstudies[i]," tests. \n\n"); print(addmargins(keepscore,2)) + } # Next i (Ntests) Number of times row method is significantly more accurate than column method for 100 tests. Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 10 0 0 10 Puniform 5 0 0 0 5 MaxLike 15 15 0 5 35 Zcurve 15 15 10 0 40 Number of times row method is significantly more accurate than column method for 250 tests. Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 10 0 0 10 Puniform 5 0 0 0 5 MaxLike 15 15 0 0 30 Zcurve 15 15 14 0 44 Number of times row method is significantly more accurate than column method for 500 tests. Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 10 0 0 10 Puniform 5 0 0 0 5 MaxLike 15 15 0 0 30 Zcurve 15 15 15 0 45 Number of times row method is significantly more accurate than column method for 1000 tests. Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 10 0 0 10 Puniform 5 0 0 0 5 MaxLike 15 15 0 0 30 Zcurve 15 15 15 0 45 Number of times row method is significantly more accurate than column method for 2000 tests. Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 10 0 0 10 Puniform 5 0 0 0 5 MaxLike 15 15 0 0 30 Zcurve 15 15 15 0 45 >