MA: Bias when adding trial

library(pwb)
library(ggplot2)

Trial design with no precision weighted bias

We identify a trial design with substantial unadjusted bias and no precision weighted bias. We obtain bias for a meta-analysis before and after including this design in the meta-analysis. We expect no increase in (absolute) bias.

Find a Simon design with p₀=0.5, p₁=0.6, type-I error-rate 0.05, power 0.9:

all.designs <- clinfun::ph2simon(pu=0.5,
                                 pa=0.6,
                                 ep1=0.05,
                                 ep2=0.1,
                                 nmax=500)
# Choose optimal Simon design:
opt.index <- which.min(all.designs$out[, "EN(p0)"])
simon.des <- all.designs$out[opt.index, ]

This design has some unadjusted bias and no precision weighted bias when true response probability theta=0.5:

  nsims <- 1e5
  theta <- 0.5
  set.seed(16)
  single.simon <- pwbSimon(theta=theta,
                                des=simon.des,
                                nsims=nsims)
  round(single.simon$ests[3:4, 2:4], 5)
#>                             bias mean.SE  emp.SE
#> All                      -0.0095 0.04376 0.03946
#> All (precision-weighted)  0.0000 0.04066      NA
  mcse.simon <- MCSEbias(single.simon$results$theta.hat)

There is unadjusted bias and no precision weighted bias.

Now simulate all other meta-analysis studies and find summary estimate for each replication. These studies all have sample size equal to the maximum sample size of the Simon design.

ma.results <- simMeta(N=simon.des["n"], theta=0.5, n.studies=4, nsims=nsims)
wtdmeans.ex <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.ex[i] <- weighted.mean(x=ma.results$theta.hat[i, ], w=1/ma.results$se[i, ]^2)

Now add the replications of the Simon design to the meta-analysis and find summary estimates again:

combined.results <- ma.results
combined.results$theta.hat <- cbind(single.simon$results$theta.hat, ma.results$theta.hat)
combined.results$se <- cbind(single.simon$results$se, ma.results$se)

wtdmeans.inc <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.inc[i] <- weighted.mean(x=combined.results$theta.hat[i, ], w=1/combined.results$se[i, ]^2)

Now find the bias in the meta-analysis, excluding and including the Simon design:

bias.ex <- mean(wtdmeans.ex-theta)
bias.ex
#> [1] 2.775542e-05
unwtd.bias.ex <- mean(rowMeans(ma.results$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.ex
#> [1] 2.645923e-05

bias.inc <- mean(wtdmeans.inc-theta)
bias.inc
#> [1] -0.0001688474
unwtd.bias.inc <- mean(rowMeans(combined.results$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.inc
#> [1] -0.001877981

abs(bias.inc)-abs(bias.ex)
#> [1] 0.0001410919

# MCSE:
mcse.ex <- MCSEbias(wtdmeans.ex)
mcse.ex * 1e5
#> [1] 5.196651
mcse.inc <- MCSEbias(wtdmeans.inc)
mcse.inc * 1e5
#> [1] 4.836795

Bias increases by almost an order of magnitude but still remains very small.

emp.SE <- apply(combined.results$theta.hat, 2, empSE) # emp SE of each column
precision <- 1/(emp.SE^2)
prec.simon <- precision[1]
ave.prec.non.ad <- mean(precision[-1])
# Precision of Simon:
prec.simon
#>          
#> 642.1294
# Mean precision of non-adaptive design studies:
ave.prec.non.ad
#> [1] 936.5774
# Fraction of Simon vs mean of non-adaptive:
prec.frac <- prec.simon/ave.prec.non.ad
prec.frac
#>           
#> 0.6856127

For simulation study 1, average precision of Simon as a fraction of the non-adaptive studies is 0.69.

Trial design with some precision weighted bias

We now do the converse: identify a trial with precision weighted bias but no unadjusted bias.

This is the case for the Simon design above when the true response probability is theta=0.3:

# Find bias for this design, for a single true response probability theta:
  theta <- 0.3
  set.seed(16)
  single.simon2 <- pwbSimon(theta=0.3,
                                des=simon.des,
                                nsims=nsims)
  round(single.simon2$ests[3:4, 2:4], 5)
#>                              bias mean.SE  emp.SE
#> All                       0.00014 0.04468 0.04491
#> All (precision-weighted) -0.00390 0.04450      NA
  mcse.simon2 <- MCSEbias(single.simon2$results$theta.hat)

There is some unadjusted bias but the precision weighted bias is more than one order of magnitude greater.

Again, simulate all other meta-analysis studies and find summary estimate for each replication. These studies all have sample size equal to the maximum sample size of the Simon design directly above.

# Summary estimates (excluding Simon design):
ma.results2 <- simMeta(N=simon.des["n"], theta=theta, n.studies=4, nsims=nsims)
wtdmeans.ex2 <- rep(NA, nsims)
for(i in 1:nsims)    wtdmeans.ex2[i] <- weighted.mean(x=ma.results2$theta.hat[i, ], w=1/ma.results2$se[i, ]^2)

Now add replications of this second Simon design to a meta-analysis:

combined.results2 <- ma.results2
combined.results2$theta.hat <- cbind(single.simon2$results$theta.hat, ma.results2$theta.hat)
combined.results2$se <- cbind(single.simon2$results$se, ma.results2$se)

# Summary estimates (including Simon design):
wtdmeans.inc2 <- rep(NA, nsims)
for(i in 1:nsims) wtdmeans.inc2[i] <- weighted.mean(x=combined.results2$theta.hat[i, ], w=1/combined.results2$se[i, ]^2)

Obtain bias excluding/including Simon:

# Bias excluding:
bias.ex2 <- mean(wtdmeans.ex2-theta)
bias.ex2
#> [1] -0.001363598
unwtd.bias.ex2 <- mean(rowMeans(ma.results2$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.ex2
#> [1] -5.763948e-05


# Bias including:
bias.inc2 <- mean(wtdmeans.inc2-theta)
bias.inc2
#> [1] -0.001617706
unwtd.bias.inc2 <- mean(rowMeans(combined.results2$theta.hat)) - theta # Bias using simple mean of estimates
unwtd.bias.inc2
#> [1] -1.876543e-05

# Absolute bias, difference:
abs(bias.inc2)-abs(bias.ex2)
#> [1] 0.0002541073

# MCSE:
mcse.ex2 <- MCSEbias(wtdmeans.ex2)
mcse.ex2 * 1e5
#> [1] 4.791798
mcse.inc2 <- MCSEbias(wtdmeans.inc2)
mcse.inc2 * 1e5
#> [1] 4.550449

Absolute bias increases only slightly, but we expect a larger increase.

Find the average precision of the non-adaptive design studies and the Simon design:

emp.SE2 <- apply(combined.results2$theta.hat, 2, empSE) # emp SE of each column
precision2 <- 1/(emp.SE2^2)
prec.simon2 <- precision2[1]
ave.prec.non.ad2 <- mean(precision2[-1])
# Precision of Simon:
prec.simon2
#>          
#> 495.8727
# Mean precision of non-adaptive design studies:
ave.prec.non.ad2
#> [1] 1109.634
# Fraction of Simon vs mean of non-adaptive:
prec.frac2 <- prec.simon2/ave.prec.non.ad2
prec.frac2
#>           
#> 0.4468794

For simulation study 2, average precision of Simon as a fraction of the non-adaptive studies is 0.45.

Summary of biases and MCSEs

simon.unwtd.bias <- c(single.simon$ests["All", "bias"], single.simon2$ests["All", "bias"])
simon.pw.bias <- c(single.simon$ests["All (precision-weighted)", "bias"], single.simon2$ests["All (precision-weighted)", "bias"])
ma.bias.ex <- c(bias.ex, bias.ex2)
ma.bias.inc <- c(bias.inc, bias.inc2)
#ma.unwtd.bias.ex <- c(unwtd.bias.ex, unwtd.bias.ex2)
#ma.unwtd.bias.inc <- c(unwtd.bias.inc, unwtd.bias.inc2) 
ma.mcse.ex <- c(mcse.ex, mcse.ex2)
ma.mcse.inc <- c(mcse.inc, mcse.inc2)
simon.mcse <- c(mcse.simon, mcse.simon2)
results <- data.frame(simon.unwtd.bias,
                      simon.mcse,
                      simon.pw.bias,
                      ma.bias.ex,
                      ma.bias.inc,
                      ma.mcse.ex,
                      ma.mcse.inc)
tab <- xtable::xtable(x=results*1e5, digits=1, align="rrrrrrrr", caption="Summary of results *1e5")
print(tab, type="html")

Summary of results *1e5
	simon.unwtd.bias	simon.mcse	simon.pw.bias	ma.bias.ex	ma.bias.inc	ma.mcse.ex	ma.mcse.inc
1	-949.6	12.5	0.5	2.8	-16.9	5.2	4.8
2	13.7	14.2	-389.9	-136.4	-161.8	4.8	4.6

Martin Law

2022-06-06

Trial design with no precision weighted bias

Trial design with some precision weighted bias

Summary of biases and MCSEs