code icon Code

A/B Test Sample Size Calculator

Calculate minimum sample size for statistically significant A/B tests

Source Code

const [baselineRate, mde, significance, power, dailyTraffic] = process.argv.slice(2);

const p1 = parseFloat(baselineRate);
const relativeEffect = parseFloat(mde);
const alpha = 1 - parseFloat(significance || "0.95");
const beta = 1 - parseFloat(power || "0.8");
const traffic = parseInt(dailyTraffic || "0", 10);

// Validate inputs
if (isNaN(p1) || p1 <= 0 || p1 >= 1) {
  console.log(JSON.stringify({ error: "baselineRate must be between 0 and 1" }));
  process.exit(1);
}
if (isNaN(relativeEffect) || relativeEffect <= 0) {
  console.log(JSON.stringify({ error: "mde must be a positive number" }));
  process.exit(1);
}

// Calculate target rate (baseline + relative lift)
const p2 = p1 * (1 + relativeEffect);

// Z-scores for common alpha and beta values
// Using approximations for standard values
function getZScore(probability) {
  // Common z-scores lookup
  const zScores = {
    0.005: 2.576, // 99.5%
    0.01: 2.326, // 99%
    0.025: 1.96, // 97.5% (two-tailed 95%)
    0.05: 1.645, // 95%
    0.1: 1.282, // 90%
    0.2: 0.842, // 80%
  };
  // Find closest match or interpolate
  const keys = Object.keys(zScores).map(Number).sort((a, b) => a - b);
  for (const key of keys) {
    if (Math.abs(key - probability) < 0.001) return zScores[key];
  }
  // Fallback approximation using inverse error function approximation
  const t = Math.sqrt(-2 * Math.log(probability));
  return t - (2.515517 + 0.802853 * t + 0.010328 * t * t) /
    (1 + 1.432788 * t + 0.189269 * t * t + 0.001308 * t * t * t);
}

const zAlpha = getZScore(alpha / 2); // Two-tailed test
const zBeta = getZScore(beta);

// Sample size formula for two proportions
// n = (zα/2 + zβ)² × (p1(1-p1) + p2(1-p2)) / (p2 - p1)²
const pooledVariance = p1 * (1 - p1) + p2 * (1 - p2);
const effectSize = Math.abs(p2 - p1);
const sampleSizePerVariant = Math.ceil(
  Math.pow(zAlpha + zBeta, 2) * pooledVariance / Math.pow(effectSize, 2)
);

const totalSampleSize = sampleSizePerVariant * 2;

// Duration estimate
let durationDays = null;
let durationWeeks = null;
if (traffic > 0) {
  // Assuming 50/50 split, we need totalSampleSize users
  durationDays = Math.ceil(totalSampleSize / traffic);
  durationWeeks = Math.ceil(durationDays / 7);
}

const result = {
  inputs: {
    baselineRate: p1,
    targetRate: Math.round(p2 * 10000) / 10000,
    relativeEffect: `${Math.round(relativeEffect * 100)}%`,
    significance: `${Math.round((1 - alpha) * 100)}%`,
    power: `${Math.round((1 - beta) * 100)}%`,
  },
  sampleSize: {
    perVariant: sampleSizePerVariant,
    total: totalSampleSize,
  },
  ...(traffic > 0 && {
    duration: {
      dailyTraffic: traffic,
      estimatedDays: durationDays,
      estimatedWeeks: durationWeeks,
      note: "Assumes 50/50 traffic split between control and variant",
    },
  }),
  methodology: {
    test: "Two-proportion z-test",
    tails: "Two-tailed",
    formula: "n = (zα/2 + zβ)² × (p1(1-p1) + p2(1-p2)) / (p2 - p1)²",
  },
};

console.log(JSON.stringify(result, null, 2));