# Study3.1.Results.txt > ################################################################################## > # AnalyzeStudy3.txt > ################################################################################## > # Analysis of small sample size data from Study 3, Poisson heterogeneity in > # sample size, Gamma in effect size, chi-squared tests > > rm(list=ls()) > options(scipen=999) # To avoid scientific notation! > # read.table chokes on large data sets > # simdata = scan("Testoutput.txt") > simdata = scan("Study3.1.Data.txt") # .1=1df, .3=3df, .5=5df Read 4082400 items > # Calculate the number of simulations. It might not be exactly 10,000. There are > # 3 values of SD(Effect size) after selection > # 5 values of k = Number of studies > # 3 values of mean power after selection > # 9 data values per line, for a total of > # 3*5*3*9 = 405 numbers produced by each simulation. > nsim = length(simdata)/405 > cat("\nAnalyzing data from",nsim," simulations \n\n") Analyzing data from 10080 simulations > nrows = length(simdata)/9; ncols = 9 > dim(simdata) = c(ncols,nrows) # Reversed: R reads by columns, rows > simdata = t(simdata) > colnames(simdata) = c("Simulation", "SD_EffectSize", "Nstudies", "PopMeanPower", + "SampleMeanPower", "Pcurve", "Puniform", "MaxLike", "Zcurve") > head(simdata) ; tail(simdata) Simulation SD_EffectSize Nstudies PopMeanPower SampleMeanPower Pcurve Puniform MaxLike Zcurve [1,] 1 0.1 100 0.25 0.2854065 0.18781770 0.2948268 0.2341038 0.2033952 [2,] 2 0.1 100 0.25 0.2537375 0.25229400 0.4090130 0.2616257 0.2688270 [3,] 3 0.1 100 0.25 0.2657720 0.24475360 0.3269224 0.2673504 0.2512300 [4,] 4 0.1 100 0.25 0.2531789 0.40281930 0.4520850 0.3751303 0.3975946 [5,] 5 0.1 100 0.25 0.2334679 0.22395450 0.3080781 0.2671308 0.2460019 [6,] 6 0.1 100 0.25 0.2302742 0.08147823 0.1403617 0.1576886 0.1344362 Simulation SD_EffectSize Nstudies PopMeanPower SampleMeanPower Pcurve Puniform MaxLike Zcurve [453595,] 31 0.3 2000 0.75 0.7540346 0.9633801 1 0.7535242 0.7682950 [453596,] 32 0.3 2000 0.75 0.7481798 0.9645911 1 0.7477931 0.7587534 [453597,] 33 0.3 2000 0.75 0.7428611 0.9622341 1 0.7447206 0.7606513 [453598,] 34 0.3 2000 0.75 0.7576436 0.9657301 1 0.7554975 0.7708686 [453599,] 35 0.3 2000 0.75 0.7465104 0.9666263 1 0.7451535 0.7466631 [453600,] 36 0.3 2000 0.75 0.7540943 0.9735199 1 0.7632264 0.7750657 > # 50 MLEs out of 1.8 million were bizarre. > simdata[,8][simdata[,8]>1] = NA # Temporary fix-up. I'm re-running the job. > # But this should not affect the results. > > # Assembing data in univariate format for nice easy tables > # This way is fast. > nlines = dim(simdata)[1] # Will be bigger for full data set. > # Four copies of the IV columns > SD_EffectSize = rep(simdata[,2],each=4) > Nstudies = rep(simdata[,3],each=4) > PopMeanPower = rep(simdata[,4],each=4) > SampleMeanPower = rep(simdata[,5],each=4) > Methods = c("Pcurve","Puniform","MaxLike","Zcurve") > Method = factor(rep(1:4,times=nlines),labels=Methods) > est = as.matrix(simdata[,6:9]) # Estimates > Estimate = as.numeric(t(est)) > # cbind(SD_EffectSize, Nstudies, PopMeanPower, SampleMeanPower, Estimate)[1:8,] > Bias = Estimate - PopMeanPower > AbsError = abs(Bias) > # Call the estimate close enough if it's within 5% of the target. > CloseEnough = numeric(length(AbsError)); CloseEnough[AbsError<0.05] = 1 > > > ######################### Sample sizes ############################# > kounts = table(Method, SD_EffectSize, PopMeanPower,Nstudies) > cat("\n Sample sizes \n\n"); print(kounts) Sample sizes , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 10080 10080 10080 Puniform 10080 10080 10080 MaxLike 10080 10080 10080 Zcurve 10080 10080 10080 > > ###################### Average Performance ###################### > estframe = aggregate(Estimate,by=list(Method, SD_EffectSize, PopMeanPower,Nstudies),FUN=mean, na.rm=T) > meanest = estframe$x; dim(meanest) = c(4,3,3,5) # Numbers of factor levels, in order > MeanEstimatedPower = kounts # To use the nice labels > for(j in 1:5) MeanEstimatedPower[,,,j] = meanest[,,,j] > cat("\n Mean Estimated Power \n\n") Mean Estimated Power > print(round(MeanEstimatedPower,3)) , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.222 0.259 0.301 Puniform 0.306 0.773 0.972 MaxLike 0.237 0.271 0.283 Zcurve 0.263 0.246 0.246 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.543 0.663 0.742 Puniform 0.608 0.927 0.996 MaxLike 0.505 0.504 0.507 Zcurve 0.539 0.527 0.523 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.821 0.924 0.960 Puniform 0.864 0.993 1.000 MaxLike 0.752 0.752 0.752 Zcurve 0.750 0.774 0.787 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.222 0.265 0.311 Puniform 0.304 0.853 0.997 MaxLike 0.236 0.269 0.282 Zcurve 0.250 0.234 0.235 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.546 0.676 0.754 Puniform 0.610 0.948 0.999 MaxLike 0.503 0.504 0.507 Zcurve 0.527 0.514 0.507 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.825 0.928 0.963 Puniform 0.867 0.996 1.000 MaxLike 0.753 0.751 0.751 Zcurve 0.752 0.769 0.779 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.224 0.271 0.316 Puniform 0.305 0.920 1.000 MaxLike 0.235 0.268 0.281 Zcurve 0.244 0.231 0.231 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.550 0.680 0.758 Puniform 0.612 0.962 1.000 MaxLike 0.503 0.502 0.506 Zcurve 0.521 0.505 0.498 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.827 0.929 0.964 Puniform 0.868 0.997 1.000 MaxLike 0.753 0.750 0.750 Zcurve 0.753 0.765 0.772 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.226 0.275 0.320 Puniform 0.304 0.974 1.000 MaxLike 0.235 0.268 0.281 Zcurve 0.240 0.229 0.230 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.553 0.683 0.761 Puniform 0.613 0.978 1.000 MaxLike 0.503 0.502 0.506 Zcurve 0.516 0.499 0.492 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.828 0.930 0.964 Puniform 0.868 0.998 1.000 MaxLike 0.752 0.750 0.750 Zcurve 0.753 0.761 0.766 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.228 0.277 0.323 Puniform 0.305 0.996 1.000 MaxLike 0.236 0.268 0.281 Zcurve 0.238 0.229 0.230 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.554 0.684 0.762 Puniform 0.613 0.992 1.000 MaxLike 0.502 0.502 0.505 Zcurve 0.512 0.495 0.488 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.829 0.931 0.965 Puniform 0.869 0.999 1.000 MaxLike 0.751 0.750 0.750 Zcurve 0.754 0.758 0.761 > > > ########################## Standard deviations ######################### > vframe = aggregate(Estimate,by=list(Method, SD_EffectSize, PopMeanPower,Nstudies),FUN=var, na.rm=T) > vest = vframe$x; dim(vest) = c(4,3,3,5) # Numbers of factor levels, in order > SDestimatedPower = kounts # To use the nice labels > for(j in 1:5) SDestimatedPower[,,,j] = sqrt(vest[,,,j]) > cat("\n Standard deviations of estimated power \n\n") Standard deviations of estimated power > print(round(SDestimatedPower,3)) , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.070 0.095 0.113 Puniform 0.089 0.188 0.077 MaxLike 0.079 0.046 0.043 Zcurve 0.069 0.062 0.060 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.074 0.086 0.085 Puniform 0.074 0.063 0.012 MaxLike 0.075 0.057 0.055 Zcurve 0.068 0.071 0.073 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.041 0.029 0.021 Puniform 0.037 0.008 0.000 MaxLike 0.064 0.046 0.045 Zcurve 0.044 0.037 0.038 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.046 0.063 0.072 Puniform 0.058 0.146 0.019 MaxLike 0.069 0.029 0.027 Zcurve 0.046 0.042 0.039 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.048 0.053 0.053 Puniform 0.048 0.040 0.003 MaxLike 0.045 0.035 0.035 Zcurve 0.047 0.047 0.046 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.026 0.017 0.012 Puniform 0.023 0.004 0.000 MaxLike 0.036 0.030 0.028 Zcurve 0.032 0.027 0.026 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.033 0.046 0.053 Puniform 0.043 0.114 0.002 MaxLike 0.064 0.021 0.020 Zcurve 0.035 0.032 0.030 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.033 0.038 0.037 Puniform 0.033 0.035 0.001 MaxLike 0.030 0.026 0.024 Zcurve 0.035 0.035 0.034 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.018 0.012 0.009 Puniform 0.016 0.003 0.000 MaxLike 0.026 0.025 0.020 Zcurve 0.025 0.021 0.020 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.024 0.033 0.037 Puniform 0.033 0.065 0.000 MaxLike 0.062 0.016 0.016 Zcurve 0.027 0.026 0.024 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.024 0.027 0.026 Puniform 0.024 0.029 0.000 MaxLike 0.022 0.019 0.018 Zcurve 0.026 0.026 0.025 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.013 0.009 0.006 Puniform 0.011 0.002 0.000 MaxLike 0.023 0.017 0.016 Zcurve 0.020 0.017 0.016 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.018 0.024 0.027 Puniform 0.031 0.021 0.000 MaxLike 0.060 0.012 0.012 Zcurve 0.022 0.021 0.020 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.017 0.019 0.018 Puniform 0.017 0.018 0.000 MaxLike 0.018 0.014 0.015 Zcurve 0.019 0.020 0.019 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.009 0.006 0.004 Puniform 0.008 0.001 0.000 MaxLike 0.019 0.017 0.010 Zcurve 0.015 0.013 0.012 > > ########################## Sample mean true power ######################### > TPframe = aggregate(SampleMeanPower,by=list(Method, SD_EffectSize, PopMeanPower,Nstudies),FUN=mean, na.rm=T) > tp = TPframe$x; dim(tp) = c(4,3,3,5) > TPOW = kounts > for(j in 1:5) TPOW[,,,j] = tp[,,,j] > cat("\n Sample mean true power \n\n") Sample mean true power > print(round(TPOW,3)) , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.250 0.249 0.249 Puniform 0.250 0.249 0.249 MaxLike 0.250 0.249 0.249 Zcurve 0.250 0.249 0.249 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.502 0.498 0.498 Puniform 0.502 0.498 0.498 MaxLike 0.502 0.498 0.498 Zcurve 0.502 0.498 0.498 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.752 0.749 0.749 Puniform 0.752 0.749 0.749 MaxLike 0.752 0.749 0.749 Zcurve 0.752 0.749 0.749 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.249 0.249 0.249 Puniform 0.249 0.249 0.249 MaxLike 0.249 0.249 0.249 Zcurve 0.249 0.249 0.249 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.501 0.499 0.498 Puniform 0.501 0.499 0.498 MaxLike 0.501 0.499 0.498 Zcurve 0.501 0.499 0.498 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.752 0.749 0.749 Puniform 0.752 0.749 0.749 MaxLike 0.752 0.749 0.749 Zcurve 0.752 0.749 0.749 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.250 0.250 0.249 Puniform 0.250 0.250 0.249 MaxLike 0.250 0.250 0.249 Zcurve 0.250 0.250 0.249 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.501 0.499 0.499 Puniform 0.501 0.499 0.499 MaxLike 0.501 0.499 0.499 Zcurve 0.501 0.499 0.499 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.752 0.750 0.749 Puniform 0.752 0.750 0.749 MaxLike 0.752 0.750 0.749 Zcurve 0.752 0.750 0.749 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.250 0.250 0.250 Puniform 0.250 0.250 0.250 MaxLike 0.250 0.250 0.250 Zcurve 0.250 0.250 0.250 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.502 0.499 0.499 Puniform 0.502 0.499 0.499 MaxLike 0.502 0.499 0.499 Zcurve 0.502 0.499 0.499 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.752 0.750 0.750 Puniform 0.752 0.750 0.750 MaxLike 0.752 0.750 0.750 Zcurve 0.752 0.750 0.750 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.250 0.250 0.250 Puniform 0.250 0.250 0.250 MaxLike 0.250 0.250 0.250 Zcurve 0.250 0.250 0.250 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.502 0.500 0.499 Puniform 0.502 0.500 0.499 MaxLike 0.502 0.500 0.499 Zcurve 0.502 0.500 0.499 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.752 0.750 0.750 Puniform 0.752 0.750 0.750 MaxLike 0.752 0.750 0.750 Zcurve 0.752 0.750 0.750 > > ###################### Bias ###################### > biasframe = aggregate(Bias,by=list(Method, SD_EffectSize, PopMeanPower,Nstudies),FUN=mean, na.rm=T) > meanbias = biasframe$x; dim(meanbias) = c(4,3,3,5) > MeanBias = kounts > for(j in 1:5) MeanBias[,,,j] = meanbias[,,,j] > cat("\n Mean Bias \n\n") Mean Bias > print(round(MeanBias,3)) , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -0.028 0.009 0.051 Puniform 0.056 0.523 0.722 MaxLike -0.013 0.021 0.033 Zcurve 0.013 -0.004 -0.004 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.043 0.163 0.242 Puniform 0.108 0.427 0.496 MaxLike 0.005 0.004 0.007 Zcurve 0.039 0.027 0.023 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.071 0.174 0.210 Puniform 0.114 0.243 0.250 MaxLike 0.002 0.002 0.002 Zcurve 0.000 0.024 0.037 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -0.028 0.015 0.061 Puniform 0.054 0.603 0.747 MaxLike -0.014 0.019 0.032 Zcurve 0.000 -0.016 -0.015 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.046 0.176 0.254 Puniform 0.110 0.448 0.499 MaxLike 0.003 0.004 0.007 Zcurve 0.027 0.014 0.007 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.075 0.178 0.213 Puniform 0.117 0.246 0.250 MaxLike 0.003 0.001 0.001 Zcurve 0.002 0.019 0.029 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -0.026 0.021 0.066 Puniform 0.055 0.670 0.750 MaxLike -0.015 0.018 0.031 Zcurve -0.006 -0.019 -0.019 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.050 0.180 0.258 Puniform 0.112 0.462 0.500 MaxLike 0.003 0.002 0.006 Zcurve 0.021 0.005 -0.002 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.077 0.179 0.214 Puniform 0.118 0.247 0.250 MaxLike 0.003 0.000 0.000 Zcurve 0.003 0.015 0.022 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -0.024 0.025 0.070 Puniform 0.054 0.724 0.750 MaxLike -0.015 0.018 0.031 Zcurve -0.010 -0.021 -0.020 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.053 0.183 0.261 Puniform 0.113 0.478 0.500 MaxLike 0.003 0.002 0.006 Zcurve 0.016 -0.001 -0.008 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.078 0.180 0.214 Puniform 0.118 0.248 0.250 MaxLike 0.002 0.000 0.000 Zcurve 0.003 0.011 0.016 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -0.022 0.027 0.073 Puniform 0.055 0.746 0.750 MaxLike -0.014 0.018 0.031 Zcurve -0.012 -0.021 -0.020 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.054 0.184 0.262 Puniform 0.113 0.492 0.500 MaxLike 0.002 0.002 0.005 Zcurve 0.012 -0.005 -0.012 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 0.079 0.181 0.215 Puniform 0.119 0.249 0.250 MaxLike 0.001 0.000 0.000 Zcurve 0.004 0.008 0.011 > > ########################## Z-tests for bias ######################### > # Calculate Z > ZforBias = kounts > for(j in 1:5) ZforBias[,,,j] = + sqrt(kounts[1,1,1,1])*(MeanEstimatedPower[,,,j] - TPOW[,,,j])/SDestimatedPower[,,,j] > cat("\n Z-tests for bias \n", + " Any Z with |Z|>4.61 is significant.\n\n") Z-tests for bias Any Z with |Z|>4.61 is significant. > print(round(ZforBias,2)) , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -39.37 11.27 46.10 Puniform 62.93 279.96 936.81 MaxLike -16.56 48.55 79.25 Zcurve 18.96 -5.05 -5.24 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 56.43 192.20 286.80 Puniform 143.40 688.32 4293.72 MaxLike 3.82 10.99 17.93 Zcurve 54.08 41.39 34.92 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 170.35 607.16 1026.24 Puniform 308.43 2911.68 55594.08 MaxLike 0.91 4.83 6.85 Zcurve -3.43 67.80 103.06 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -60.98 25.44 85.91 Puniform 95.12 414.31 3934.49 MaxLike -19.75 67.70 121.30 Zcurve 2.38 -34.96 -36.27 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 93.66 332.92 487.74 Puniform 229.87 1123.29 19407.34 MaxLike 3.98 13.53 23.95 Zcurve 55.01 33.05 19.40 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 284.33 1025.39 1750.13 Puniform 501.03 5994.07 600926.61 MaxLike 3.42 4.87 7.26 Zcurve -0.40 73.16 113.74 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -77.68 46.00 126.80 Puniform 128.85 590.86 33752.64 MaxLike -22.52 88.64 158.78 Zcurve -16.39 -59.21 -61.43 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 145.27 477.77 706.93 Puniform 331.56 1336.08 88682.51 MaxLike 5.94 13.23 28.65 Zcurve 57.55 17.36 -3.82 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 412.86 1478.48 2515.92 Puniform 711.70 8331.83 3588980.69 MaxLike 2.61 0.47 6.25 Zcurve 4.26 72.27 111.96 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -98.92 76.62 190.60 Puniform 164.33 1124.57 6187465.33 MaxLike -23.38 115.07 193.60 Zcurve -35.96 -78.87 -82.26 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 209.98 691.42 1013.85 Puniform 467.16 1679.45 940944.16 MaxLike 3.34 17.55 37.07 Zcurve 55.05 0.23 -28.51 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 596.08 2126.69 3651.13 Puniform 1028.65 11280.21 37124647.93 MaxLike -0.40 2.51 4.93 Zcurve 7.36 68.56 105.17 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve -124.67 114.74 277.44 Puniform 176.45 3614.06 Inf MaxLike -23.45 152.47 267.20 Zcurve -54.88 -97.40 -103.25 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 307.89 979.69 1445.80 Puniform 671.43 2761.86 128853293.70 MaxLike -1.30 15.46 38.66 Zcurve 51.06 -24.64 -58.37 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 865.89 2995.99 5240.78 Puniform 1474.82 18182.30 17850492945.40 MaxLike -3.18 0.60 8.78 Zcurve 11.50 62.93 92.54 > > # Protect the 240 tests in this table with Bonferroni > # Two-tailed at the 0.001 significance level > # 0.0005/240 = 0.000002083333, and qnorm(0.000002083333) = -4.602891 > # So any Z with |Z|>4.61 is significant. > > ###################### Is estimate close enough? ############# > > # Get table of counts in a different order, for labels > k2 = table(Method, Nstudies, PopMeanPower, SD_EffectSize) > > closeframe = + aggregate(CloseEnough,by=list(Method, Nstudies, PopMeanPower, SD_EffectSize),FUN=mean, na.rm=T) > klose = closeframe$x; dim(klose) = c(4,5,3,3) > ProportionWithin.05 = k2 > for(j in 1:3) ProportionWithin.05[,,,j] = klose[,,,j] > cat("\n Proportion of estimates within 0.05 of target \n\n") Proportion of estimates within 0.05 of target > print(round(ProportionWithin.05,3)) , , PopMeanPower = 0.25, SD_EffectSize = 0.1 Nstudies Method 100 250 500 1000 2000 Pcurve 0.458 0.610 0.751 0.858 0.947 Puniform 0.381 0.457 0.459 0.456 0.442 MaxLike 0.615 0.795 0.889 0.915 0.924 Zcurve 0.531 0.723 0.846 0.926 0.974 , , PopMeanPower = 0.5, SD_EffectSize = 0.1 Nstudies Method 100 250 500 1000 2000 Pcurve 0.414 0.495 0.495 0.448 0.398 Puniform 0.193 0.103 0.038 0.006 0.000 MaxLike 0.545 0.762 0.912 0.979 0.997 Zcurve 0.455 0.622 0.779 0.897 0.973 , , PopMeanPower = 0.75, SD_EffectSize = 0.1 Nstudies Method 100 250 500 1000 2000 Pcurve 0.273 0.165 0.080 0.019 0.001 Puniform 0.055 0.006 0.000 0.000 0.000 MaxLike 0.661 0.877 0.971 0.996 0.999 Zcurve 0.752 0.879 0.946 0.989 0.999 , , PopMeanPower = 0.25, SD_EffectSize = 0.2 Nstudies Method 100 250 500 1000 2000 Pcurve 0.387 0.560 0.687 0.779 0.839 Puniform 0.011 0.000 0.000 0.000 0.000 MaxLike 0.689 0.858 0.939 0.983 0.993 Zcurve 0.575 0.718 0.806 0.877 0.933 , , PopMeanPower = 0.5, SD_EffectSize = 0.2 Nstudies Method 100 250 500 1000 2000 Pcurve 0.086 0.016 0.001 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.621 0.839 0.957 0.994 1.000 Zcurve 0.472 0.683 0.849 0.949 0.985 , , PopMeanPower = 0.75, SD_EffectSize = 0.2 Nstudies Method 100 250 500 1000 2000 Pcurve 0.001 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.727 0.920 0.985 0.999 1.000 Zcurve 0.701 0.861 0.955 0.988 0.999 , , PopMeanPower = 0.25, SD_EffectSize = 0.3 Nstudies Method 100 250 500 1000 2000 Pcurve 0.321 0.394 0.378 0.307 0.193 Puniform 0.001 0.000 0.000 0.000 0.000 MaxLike 0.646 0.759 0.845 0.911 0.958 Zcurve 0.596 0.759 0.834 0.902 0.956 , , PopMeanPower = 0.5, SD_EffectSize = 0.3 Nstudies Method 100 250 500 1000 2000 Pcurve 0.022 0.001 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.633 0.848 0.954 0.996 0.999 Zcurve 0.473 0.719 0.864 0.948 0.984 , , PopMeanPower = 0.75, SD_EffectSize = 0.3 Nstudies Method 100 250 500 1000 2000 Pcurve 0.000 0.000 0.000 0.000 0.000 Puniform 0.000 0.000 0.000 0.000 0.000 MaxLike 0.733 0.925 0.988 1.000 1.000 Zcurve 0.567 0.785 0.922 0.985 0.999 > > ###################### Mean Absolute Error ###################### > # Take a look at marginals to see roughly who won overall > cat(" Marginal means for method \n") Marginal means for method > print(aggregate(round(100*AbsError,2),by=list(Method),FUN=mean, na.rm=T)) Group.1 x 1 Pcurve 12.446554 2 Puniform 34.965469 3 MaxLike 2.589202 4 Zcurve 3.052147 > > errframe = aggregate(AbsError,by=list(Method, SD_EffectSize, PopMeanPower,Nstudies),FUN=mean, na.rm=T) > meanerr = 100*errframe$x; dim(meanerr) = c(4,3,3,5) > MeanAbsoluteError = kounts > for(j in 1:5) MeanAbsoluteError[,,,j] = meanerr[,,,j] > cat("\n Mean Absolute Error of Prediction \n\n") Mean Absolute Error of Prediction > print(round(MeanAbsoluteError,2)) , , PopMeanPower = 0.25, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 6.16 7.63 9.75 Puniform 8.24 52.31 72.18 MaxLike 5.59 3.95 4.34 Zcurve 5.54 4.97 4.79 , , PopMeanPower = 0.5, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 7.00 16.67 24.25 Puniform 11.40 42.72 49.61 MaxLike 5.55 4.54 4.43 Zcurve 6.39 6.16 6.18 , , PopMeanPower = 0.75, Nstudies = 100 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 7.34 17.41 20.98 Puniform 11.43 24.32 24.99 MaxLike 4.32 3.63 3.60 Zcurve 3.48 3.73 4.57 , , PopMeanPower = 0.25, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 4.44 5.17 7.62 Puniform 6.41 60.26 74.68 MaxLike 4.20 2.71 3.45 Zcurve 3.70 3.64 3.40 , , PopMeanPower = 0.5, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 5.51 17.63 25.44 Puniform 11.07 44.84 49.94 MaxLike 3.44 2.84 2.80 Zcurve 4.42 3.96 3.73 , , PopMeanPower = 0.75, Nstudies = 250 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 7.48 17.79 21.27 Puniform 11.70 24.59 25.00 MaxLike 2.66 2.29 2.26 Zcurve 2.59 2.77 3.32 , , PopMeanPower = 0.25, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 3.45 3.99 7.06 Puniform 5.80 67.05 74.99 MaxLike 3.42 2.24 3.18 Zcurve 2.83 3.14 2.97 , , PopMeanPower = 0.5, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 5.19 17.98 25.80 Puniform 11.19 46.15 49.99 MaxLike 2.37 2.05 1.99 Zcurve 3.28 2.82 2.69 , , PopMeanPower = 0.75, Nstudies = 500 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 7.67 17.92 21.37 Puniform 11.82 24.71 25.00 MaxLike 1.86 1.65 1.60 Zcurve 2.04 2.11 2.52 , , PopMeanPower = 0.25, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 2.82 3.27 7.09 Puniform 5.46 72.43 75.00 MaxLike 2.93 1.95 3.16 Zcurve 2.36 2.83 2.67 , , PopMeanPower = 0.5, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 5.31 18.28 26.07 Puniform 11.28 47.82 50.00 MaxLike 1.71 1.43 1.43 Zcurve 2.46 2.05 2.10 , , PopMeanPower = 0.75, Nstudies = 1000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 7.78 18.01 21.43 Puniform 11.84 24.83 25.00 MaxLike 1.33 1.15 1.14 Zcurve 1.61 1.61 1.86 , , PopMeanPower = 0.25, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 2.39 2.96 7.30 Puniform 5.46 74.62 75.00 MaxLike 2.49 1.82 3.15 Zcurve 2.05 2.54 2.42 , , PopMeanPower = 0.5, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 5.43 18.41 26.16 Puniform 11.31 49.19 50.00 MaxLike 1.22 1.01 1.07 Zcurve 1.80 1.65 1.79 , , PopMeanPower = 0.75, Nstudies = 2000 SD_EffectSize Method 0.1 0.2 0.3 Pcurve 7.88 18.08 21.46 Puniform 11.90 24.94 25.00 MaxLike 0.94 0.84 0.79 Zcurve 1.27 1.22 1.32 > > > ################################################################################## > # Need to do all 6 pairwise matched t-tests for mean absolute error, for each > # combination of Nstudies and PopMeanPower, > # with a Bonferroni correction for 120 tests in the table. > # alpha = 0.001; a = alpha/120; critZ = qnorm(1-a/2); critZ # 4.456436 > # Return to simdata, multivariate format. > ###################################################################################### > > # Bonferroni correct at joint 0.001 levl for 6 pairwise comparisons within > # each combination of > # 3 values of SD(Effect size) after selection > # 5 values of k = Number of studies > # 3 values of mean power after selection > # 6*45 = 270 > a = 0.001/270; critZ = qnorm(1-a/2) # 4.627352 > > ae = simdata[,6:9] # The estimates > sdes = simdata[,2]; kk = simdata[,3]; pow = simdata[,4]; Index = 1:nlines > for(kol in 1:4) ae[,kol] = abs(ae[,kol]-pow) # Now they are absolute errors. > > nstudies = unique(Nstudies); truepower = unique(PopMeanPower) > sd_effectsize = unique(SD_EffectSize) > pairwise = matrix(0,4,4); rownames(pairwise) = colnames(pairwise) = Methods > keepscore = pairwise # Count winners in this matrix > > # This does not have to be efficient, and it isn't. > > cat("\n\n Matched Z-tests for pairwise differences in mean absolute error. \n", + "Positive value means the row method is less accurate on average. \n", + "Critical value Bonferroni protected at the 0.001 level is",critZ,".\n\n") Matched Z-tests for pairwise differences in mean absolute error. Positive value means the row method is less accurate on average. Critical value Bonferroni protected at the 0.001 level is 4.627352 . > > for(h in 1:3) # Looping over sd of effect size + { + for(i in 1:5) # Looping over Nstudies + { + for(j in 1:3) # Looping over PopMeanPower + { + cat("\n SD_EffectSize =",sd_effectsize[h]," Nstudies =",nstudies[i]," and PopMeanPower =",truepower[j],"\n\n") + In = (sdes==sd_effectsize[h]) * (kk==nstudies[i]) * (pow==truepower[j]) + pick = Index[In==1] + # Fill matrix of pairwise tests in a double loop + for(K in 1:4) + { + for(L in 1:4) + { + differ = (ae[,K]-ae[,L])[pick] + Z = 0 + if(K != L) Z = t.test(differ)$statistic + if(K < L) pairwise[K,L] = Z + if(Z < -critZ) keepscore[K,L] = keepscore[K,L]+1 + } # Next L + } # Next K + print(round(pairwise[1:3,2:4],2)) + } # Next j (PopMeanPower) + } # Next i (Nstudies) + } # Next h (SD_EffectSize) SD_EffectSize = 0.1 Nstudies = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -27.89 9.67 13.93 Puniform 0.00 33.71 43.60 MaxLike 0.00 0.00 0.96 SD_EffectSize = 0.1 Nstudies = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -81.97 29.47 18.29 Puniform 0.00 73.35 79.33 MaxLike 0.00 0.00 -17.73 SD_EffectSize = 0.1 Nstudies = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -166.6 52.16 84.23 Puniform 0.0 116.85 169.38 MaxLike 0.0 0.00 19.27 SD_EffectSize = 0.1 Nstudies = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -31.79 4.03 22.84 Puniform 0.00 31.07 53.72 MaxLike 0.00 0.00 8.66 SD_EffectSize = 0.1 Nstudies = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -161.92 54.74 37.43 Puniform 0.00 139.54 141.82 MaxLike 0.00 0.00 -32.72 SD_EffectSize = 0.1 Nstudies = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -273.38 136.81 152.46 Puniform 0.00 264.43 300.07 MaxLike 0.00 0.00 3.15 SD_EffectSize = 0.1 Nstudies = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -43.78 0.54 24.38 Puniform 0.00 35.21 65.66 MaxLike 0.00 0.00 10.27 SD_EffectSize = 0.1 Nstudies = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -279.36 89.75 72.55 Puniform 0.00 233.03 230.41 MaxLike 0.00 0.00 -44.00 SD_EffectSize = 0.1 Nstudies = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -376.37 231.83 243.06 Puniform 0.00 409.43 436.66 MaxLike 0.00 0.00 -9.39 SD_EffectSize = 0.1 Nstudies = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -57.96 -1.83 22.36 Puniform 0.00 38.35 78.55 MaxLike 0.00 0.00 9.82 SD_EffectSize = 0.1 Nstudies = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -406.9 139.53 127.34 Puniform 0.0 355.77 354.96 MaxLike 0.0 0.00 -44.72 SD_EffectSize = 0.1 Nstudies = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -515.12 299.55 376.41 Puniform 0.00 497.32 636.24 MaxLike 0.00 0.00 -14.30 SD_EffectSize = 0.1 Nstudies = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -76.4 -1.64 19.95 Puniform 0.0 46.40 94.18 MaxLike 0.0 0.00 7.73 SD_EffectSize = 0.1 Nstudies = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -555.23 202.54 210.06 Puniform 0.00 483.20 525.74 MaxLike 0.00 0.00 -36.87 SD_EffectSize = 0.1 Nstudies = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -707.43 372.98 563.62 Puniform 0.00 589.85 892.67 MaxLike 0.00 0.00 -18.66 SD_EffectSize = 0.2 Nstudies = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -234.23 85.99 52.00 Puniform 0.00 261.33 246.33 MaxLike 0.00 0.00 -29.01 SD_EffectSize = 0.2 Nstudies = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -355.37 146.41 140.99 Puniform 0.00 523.24 494.83 MaxLike 0.00 0.00 -52.76 SD_EffectSize = 0.2 Nstudies = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -270.7 314.83 448.41 Puniform 0.0 688.78 829.06 MaxLike 0.0 0.00 -3.85 SD_EffectSize = 0.2 Nstudies = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -372.46 81.04 33.08 Puniform 0.00 397.64 376.89 MaxLike 0.00 0.00 -28.27 SD_EffectSize = 0.2 Nstudies = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -510 264.16 251.22 Puniform 0 921.68 842.23 MaxLike 0 0.00 -56.33 SD_EffectSize = 0.2 Nstudies = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -421.74 584.05 790.30 Puniform 0.00 1165.23 1159.71 MaxLike 0.00 0.00 -23.48 SD_EffectSize = 0.2 Nstudies = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -545.72 74.98 21.09 Puniform 0.00 570.68 545.87 MaxLike 0.00 0.00 -30.22 SD_EffectSize = 0.2 Nstudies = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -631.41 396.32 364.72 Puniform 0.00 1149.54 1080.63 MaxLike 0.00 0.00 -46.41 SD_EffectSize = 0.2 Nstudies = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -590.36 714.48 1102.49 Puniform 0.00 1231.66 1505.95 MaxLike 0.00 0.00 -22.36 SD_EffectSize = 0.2 Nstudies = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1026.33 73.11 12.99 Puniform 0.00 1080.79 1033.46 MaxLike 0.00 0.00 -34.05 SD_EffectSize = 0.2 Nstudies = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -811.95 599.61 508.93 Puniform 0.00 1505.32 1417.03 MaxLike 0.00 0.00 -43.44 SD_EffectSize = 0.2 Nstudies = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -815.43 1063.33 1449.70 Puniform 0.00 1803.54 1931.27 MaxLike 0.00 0.00 -30.55 SD_EffectSize = 0.2 Nstudies = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -2549.09 78.47 14.97 Puniform 0.00 3143.52 2752.17 MaxLike 0.00 0.00 -32.35 SD_EffectSize = 0.2 Nstudies = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -1220.94 865.89 677.26 Puniform 0.00 2370.01 2204.37 MaxLike 0.00 0.00 -47.41 SD_EffectSize = 0.2 Nstudies = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -1128.02 1046.08 1918.91 Puniform 0.00 1570.65 2539.54 MaxLike 0.00 0.00 -22.92 SD_EffectSize = 0.3 Nstudies = 100 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -606.14 95.86 68.10 Puniform 0.00 845.55 767.94 MaxLike 0.00 0.00 -11.33 SD_EffectSize = 0.3 Nstudies = 100 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -313.75 226.73 215.88 Puniform 0.00 1282.99 948.20 MaxLike 0.00 0.00 -65.64 SD_EffectSize = 0.3 Nstudies = 100 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -196.32 468.97 691.92 Puniform 0.00 776.87 765.72 MaxLike 0.00 0.00 -31.35 SD_EffectSize = 0.3 Nstudies = 250 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1155.05 98.30 64.08 Puniform 0.00 2518.78 2230.92 MaxLike 0.00 0.00 1.29 SD_EffectSize = 0.3 Nstudies = 250 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -470.17 422.74 383.35 Puniform 0.00 2160.53 1644.89 MaxLike 0.00 0.00 -51.93 SD_EffectSize = 0.3 Nstudies = 250 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -304.65 851.35 1080.19 Puniform 0.00 1330.32 1060.56 MaxLike 0.00 0.00 -46.82 SD_EffectSize = 0.3 Nstudies = 500 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1485.33 111.37 72.32 Puniform 0.00 3957.36 3585.83 MaxLike 0.00 0.00 6.57 SD_EffectSize = 0.3 Nstudies = 500 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -660.62 653.91 527.25 Puniform 0.00 3150.22 2363.15 MaxLike 0.00 0.00 -42.81 SD_EffectSize = 0.3 Nstudies = 500 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -425.79 1275.46 1439.24 Puniform 0.00 1930.49 1404.09 MaxLike 0.00 0.00 -53.42 SD_EffectSize = 0.3 Nstudies = 1000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -1880.43 137.32 97.61 Puniform 0.00 4596.99 4343.05 MaxLike 0.00 0.00 17.88 SD_EffectSize = 0.3 Nstudies = 1000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -928.13 994.62 706.80 Puniform 0.00 4076.68 3084.34 MaxLike 0.00 0.00 -39.37 SD_EffectSize = 0.3 Nstudies = 1000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -607.06 1580.51 1874.50 Puniform 0.00 2121.34 1861.44 MaxLike 0.00 0.00 -49.45 SD_EffectSize = 0.3 Nstudies = 2000 and PopMeanPower = 0.25 Puniform MaxLike Zcurve Pcurve -2567.05 195.88 145.27 Puniform 0.00 6467.28 4993.26 MaxLike 0.00 0.00 33.99 SD_EffectSize = 0.3 Nstudies = 2000 and PopMeanPower = 0.5 Puniform MaxLike Zcurve Pcurve -1313.25 1349.82 925.76 Puniform 0.00 4076.03 3826.90 MaxLike 0.00 0.00 -40.48 SD_EffectSize = 0.3 Nstudies = 2000 and PopMeanPower = 0.75 Puniform MaxLike Zcurve Pcurve -861.73 2836.52 2509.29 Puniform 0.00 4000.24 2549.19 MaxLike 0.00 0.00 -57.23 > > cat("\nTotal number of significant comparisons: \n") Total number of significant comparisons: > print(sum(keepscore)) [1] 262 > > cat("\n\n Number of times row method is significantly more accurate than column method \n\n"); print(addmargins(keepscore,2)) Number of times row method is significantly more accurate than column method Pcurve Puniform MaxLike Zcurve Sum Pcurve 0 45 0 0 45 Puniform 0 0 0 0 0 MaxLike 41 45 0 33 119 Zcurve 45 45 8 0 98