MA: Bias when adding trial
Martin Law
2022-06-06
MA_add_trial.Rmd
Trial design with no precision weighted bias
We identify a trial design with substantial unadjusted bias and no precision weighted bias. We obtain bias for a meta-analysis before and after including this design in the meta-analysis. We expect no increase in (absolute) bias.
Find a Simon design with p0=0.5, p1=0.6, type-I error-rate 0.05, power 0.9:
all.designs <- clinfun::ph2simon(pu=0.5,
pa=0.6,
ep1=0.05,
ep2=0.1,
nmax=500)
# Choose optimal Simon design:
opt.index <- which.min(all.designs$out[, "EN(p0)"])
simon.des <- all.designs$out[opt.index, ]
This design has some unadjusted bias and no precision weighted bias when true response probability theta=0.5:
nsims <- 1e5
theta <- 0.5
set.seed(16)
single.simon <- pwbSimon(theta=theta,
des=simon.des,
nsims=nsims)
round(single.simon$ests[3:4, 2:4], 5)
#> bias mean.SE emp.SE
#> All -0.0095 0.04376 0.03946
#> All (precision-weighted) 0.0000 0.04066 NA
mcse.simon <- MCSEbias(single.simon$results$theta.hat)
There is unadjusted bias and no precision weighted bias.
Now simulate all other meta-analysis studies and find summary estimate for each replication. These studies all have sample size equal to the maximum sample size of the Simon design.
ma.results <- simMeta(N=simon.des["n"], theta=0.5, n.studies=4, nsims=nsims)
wtdmeans.ex <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.ex[i] <- weighted.mean(x=ma.results$theta.hat[i, ], w=1/ma.results$se[i, ]^2)
Now add the replications of the Simon design to the meta-analysis and find summary estimates again:
combined.results <- ma.results
combined.results$theta.hat <- cbind(single.simon$results$theta.hat, ma.results$theta.hat)
combined.results$se <- cbind(single.simon$results$se, ma.results$se)
wtdmeans.inc <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.inc[i] <- weighted.mean(x=combined.results$theta.hat[i, ], w=1/combined.results$se[i, ]^2)
Now find the bias in the meta-analysis, excluding and including the Simon design:
bias.ex <- mean(wtdmeans.ex-theta)
bias.ex
#> [1] 2.775542e-05
unwtd.bias.ex <- mean(rowMeans(ma.results$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.ex
#> [1] 2.645923e-05
bias.inc <- mean(wtdmeans.inc-theta)
bias.inc
#> [1] -0.0001688474
unwtd.bias.inc <- mean(rowMeans(combined.results$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.inc
#> [1] -0.001877981
abs(bias.inc)-abs(bias.ex)
#> [1] 0.0001410919
# MCSE:
mcse.ex <- MCSEbias(wtdmeans.ex)
mcse.ex * 1e5
#> [1] 5.196651
mcse.inc <- MCSEbias(wtdmeans.inc)
mcse.inc * 1e5
#> [1] 4.836795
Bias increases by almost an order of magnitude but still remains very small.
emp.SE <- apply(combined.results$theta.hat, 2, empSE) # emp SE of each column
precision <- 1/(emp.SE^2)
prec.simon <- precision[1]
ave.prec.non.ad <- mean(precision[-1])
# Precision of Simon:
prec.simon
#>
#> 642.1294
# Mean precision of non-adaptive design studies:
ave.prec.non.ad
#> [1] 936.5774
# Fraction of Simon vs mean of non-adaptive:
prec.frac <- prec.simon/ave.prec.non.ad
prec.frac
#>
#> 0.6856127
For simulation study 1, average precision of Simon as a fraction of the non-adaptive studies is 0.69.
Trial design with some precision weighted bias
We now do the converse: identify a trial with precision weighted bias but no unadjusted bias.
This is the case for the Simon design above when the true response probability is theta=0.3:
# Find bias for this design, for a single true response probability theta:
theta <- 0.3
set.seed(16)
single.simon2 <- pwbSimon(theta=0.3,
des=simon.des,
nsims=nsims)
round(single.simon2$ests[3:4, 2:4], 5)
#> bias mean.SE emp.SE
#> All 0.00014 0.04468 0.04491
#> All (precision-weighted) -0.00390 0.04450 NA
mcse.simon2 <- MCSEbias(single.simon2$results$theta.hat)
There is some unadjusted bias but the precision weighted bias is more than one order of magnitude greater.
Again, simulate all other meta-analysis studies and find summary estimate for each replication. These studies all have sample size equal to the maximum sample size of the Simon design directly above.
# Summary estimates (excluding Simon design):
ma.results2 <- simMeta(N=simon.des["n"], theta=theta, n.studies=4, nsims=nsims)
wtdmeans.ex2 <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.ex2[i] <- weighted.mean(x=ma.results2$theta.hat[i, ], w=1/ma.results2$se[i, ]^2)
Now add replications of this second Simon design to a meta-analysis:
combined.results2 <- ma.results2
combined.results2$theta.hat <- cbind(single.simon2$results$theta.hat, ma.results2$theta.hat)
combined.results2$se <- cbind(single.simon2$results$se, ma.results2$se)
# Summary estimates (including Simon design):
wtdmeans.inc2 <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.inc2[i] <- weighted.mean(x=combined.results2$theta.hat[i, ], w=1/combined.results2$se[i, ]^2)
Obtain bias excluding/including Simon:
# Bias excluding:
bias.ex2 <- mean(wtdmeans.ex2-theta)
bias.ex2
#> [1] -0.001363598
unwtd.bias.ex2 <- mean(rowMeans(ma.results2$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.ex2
#> [1] -5.763948e-05
# Bias including:
bias.inc2 <- mean(wtdmeans.inc2-theta)
bias.inc2
#> [1] -0.001617706
unwtd.bias.inc2 <- mean(rowMeans(combined.results2$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.inc2
#> [1] -1.876543e-05
# Absolute bias, difference:
abs(bias.inc2)-abs(bias.ex2)
#> [1] 0.0002541073
# MCSE:
mcse.ex2 <- MCSEbias(wtdmeans.ex2)
mcse.ex2 * 1e5
#> [1] 4.791798
mcse.inc2 <- MCSEbias(wtdmeans.inc2)
mcse.inc2 * 1e5
#> [1] 4.550449
Absolute bias increases only slightly, but we expect a larger increase.
Find the average precision of the non-adaptive design studies and the Simon design:
emp.SE2 <- apply(combined.results2$theta.hat, 2, empSE) # emp SE of each column
precision2 <- 1/(emp.SE2^2)
prec.simon2 <- precision2[1]
ave.prec.non.ad2 <- mean(precision2[-1])
# Precision of Simon:
prec.simon2
#>
#> 495.8727
# Mean precision of non-adaptive design studies:
ave.prec.non.ad2
#> [1] 1109.634
# Fraction of Simon vs mean of non-adaptive:
prec.frac2 <- prec.simon2/ave.prec.non.ad2
prec.frac2
#>
#> 0.4468794
For simulation study 2, average precision of Simon as a fraction of the non-adaptive studies is 0.45.
Summary of biases and MCSEs
simon.unwtd.bias <- c(single.simon$ests["All", "bias"], single.simon2$ests["All", "bias"])
simon.pw.bias <- c(single.simon$ests["All (precision-weighted)", "bias"], single.simon2$ests["All (precision-weighted)", "bias"])
ma.bias.ex <- c(bias.ex, bias.ex2)
ma.bias.inc <- c(bias.inc, bias.inc2)
#ma.unwtd.bias.ex <- c(unwtd.bias.ex, unwtd.bias.ex2)
#ma.unwtd.bias.inc <- c(unwtd.bias.inc, unwtd.bias.inc2)
ma.mcse.ex <- c(mcse.ex, mcse.ex2)
ma.mcse.inc <- c(mcse.inc, mcse.inc2)
simon.mcse <- c(mcse.simon, mcse.simon2)
results <- data.frame(simon.unwtd.bias,
simon.mcse,
simon.pw.bias,
ma.bias.ex,
ma.bias.inc,
ma.mcse.ex,
ma.mcse.inc)
tab <- xtable::xtable(x=results*1e5, digits=1, align="rrrrrrrr", caption="Summary of results *1e5")
print(tab, type="html")
simon.unwtd.bias | simon.mcse | simon.pw.bias | ma.bias.ex | ma.bias.inc | ma.mcse.ex | ma.mcse.inc | |
---|---|---|---|---|---|---|---|
1 | -949.6 | 12.5 | 0.5 | 2.8 | -16.9 | 5.2 | 4.8 |
2 | 13.7 | 14.2 | -389.9 | -136.4 | -161.8 | 4.8 | 4.6 |