A/B Test Analysis Pipeline
Pipeline Architecture
flowchart LR
subgraph Raw["Raw Data"]
EV[Events Stream\nKafka]
EX[Exposure Log\nClickHouse]
end
subgraph Processing["Processing Layers"]
direction TB
D[Deduplication\nIdempotency keys]
J[Assignment Join\nLeft-join events to exposures]
M[Metric Aggregation\nGroup by variant]
S[Statistical Tests\nZ-test, t-test]
C[CUPED Adjustment\nVariance reduction]
end
subgraph Output["Results"]
RD[Results Database]
DS[Dashboard]
AL[Alerts]
end
EV & EX --> D --> J --> M --> C --> S --> RD
RD --> DS & AL
Data Pipeline Implementation
Event-to-Assignment Join
The core of the analysis is joining user events to their experiment assignments. This must be a left join on first exposure — only events after a user was first exposed to an experiment should be counted.
-- ClickHouse: compute experiment metrics for a date range
-- Step 1: Get first exposure per user per experiment
WITH first_exposures AS (
SELECT
experiment_id,
user_id,
variant_id,
min(timestamp) AS first_exposed_at
FROM analytics.experiment_exposures
WHERE
experiment_id = {experimentId: String}
AND timestamp >= {startDate: DateTime}
AND timestamp <= {endDate: DateTime}
GROUP BY experiment_id, user_id, variant_id
),
-- Step 2: Get conversion events AFTER first exposure
post_exposure_events AS (
SELECT
e.experiment_id,
e.user_id,
e.variant_id,
e.first_exposed_at,
-- Aggregate metrics per user
count(CASE WHEN t.event = 'purchase_completed' THEN 1 END) AS purchase_count,
sum(CASE WHEN t.event = 'purchase_completed'
THEN JSONExtractFloat(t.properties, 'revenue')
ELSE 0 END) AS revenue,
countIf(t.event = 'page_viewed') AS pageviews,
max(CASE WHEN t.event = 'purchase_completed' THEN 1 ELSE 0 END) AS converted
FROM first_exposures e
LEFT JOIN tracking.events t
ON e.user_id = t.user_id
AND t.timestamp >= e.first_exposed_at
AND t.timestamp <= {endDate: DateTime}
GROUP BY
e.experiment_id, e.user_id, e.variant_id, e.first_exposed_at
)
-- Step 3: Aggregate by variant
SELECT
variant_id,
count(DISTINCT user_id) AS users,
sum(purchase_count) AS total_purchases,
avg(converted) AS conversion_rate,
sum(revenue) / count(DISTINCT user_id) AS revenue_per_user,
avg(revenue) AS avg_revenue_per_converter,
varPop(converted) AS conversion_variance,
varPop(revenue) AS revenue_variance,
quantile(0.50)(pageviews) AS median_pageviews,
quantile(0.95)(pageviews) AS p95_pageviews
FROM post_exposure_events
GROUP BY variant_id
ORDER BY variant_id;
TypeScript Analysis Engine
// src/analysis/experiment-analyzer.ts
import type { ClickHouseClient } from '@clickhouse/client';
interface ExperimentMetrics {
variantId: string;
users: number;
conversionRate: number;
conversionVariance: number;
revenuePerUser: number;
revenueVariance: number;
totalRevenue: number;
}
interface AnalysisResult {
experimentId: string;
status: 'significant' | 'not-significant' | 'insufficient-data';
controlMetrics: ExperimentMetrics;
treatmentMetrics: ExperimentMetrics[];
primaryMetric: MetricComparison;
secondaryMetrics: MetricComparison[];
guardrailViolations: GuardrailViolation[];
segmentAnalysis?: SegmentResult[];
runDate: string;
}
interface MetricComparison {
metricName: string;
control: { mean: number; ci95: [number, number] };
treatment: { mean: number; ci95: [number, number] };
absoluteChange: number;
relativeChange: number;
pValue: number;
isSignificant: boolean;
practicallySignificant: boolean; // Effect size above minimum meaningful threshold
}
interface GuardrailViolation {
metricName: string;
controlValue: number;
treatmentValue: number;
relativeChange: number;
threshold: number;
severity: 'warning' | 'critical';
}
export class ExperimentAnalyzer {
constructor(private clickhouse: ClickHouseClient) {}
async analyze(
experimentId: string,
config: {
startDate: Date;
endDate: Date;
primaryMetric: string;
secondaryMetrics: string[];
guardrails: Array<{
metric: string;
maxDegradation: number;
severity: 'warning' | 'critical';
}>;
minPracticalEffect: number; // Minimum relative change that matters
}
): Promise<AnalysisResult> {
// Fetch raw metrics from ClickHouse
const metrics = await this.fetchMetrics(experimentId, config.startDate, config.endDate);
const control = metrics.find((m) => m.variantId === 'control');
if (!control) throw new Error('No control variant found');
const treatments = metrics.filter((m) => m.variantId !== 'control');
// Check for sufficient sample size
const totalUsers = metrics.reduce((s, m) => s + m.users, 0);
if (totalUsers < 100) {
return {
experimentId,
status: 'insufficient-data',
controlMetrics: control,
treatmentMetrics: treatments,
primaryMetric: this.emptyComparison(config.primaryMetric),
secondaryMetrics: [],
guardrailViolations: [],
runDate: new Date().toISOString(),
};
}
// Compute primary metric comparison for each treatment
const primaryComparisons = treatments.map((treatment) =>
this.compareMetric(
config.primaryMetric,
control,
treatment,
config.minPracticalEffect
)
);
// Compute secondary metrics
const secondaryComparisons = config.secondaryMetrics.flatMap((metric) =>
treatments.map((treatment) =>
this.compareMetric(metric, control, treatment, 0)
)
);
// Apply Bonferroni correction for multiple comparisons
const allPValues = [...primaryComparisons, ...secondaryComparisons].map(
(c) => c.pValue
);
const correctedAlpha = 0.05 / allPValues.length;
// Check guardrails
const violations = this.checkGuardrails(config.guardrails, metrics);
// Determine overall status
const isSignificant = primaryComparisons.some(
(c) => c.pValue < correctedAlpha && c.practicallySignificant
);
const hasViolation = violations.some((v) => v.severity === 'critical');
return {
experimentId,
status:
hasViolation ? 'not-significant'
: isSignificant ? 'significant'
: 'not-significant',
controlMetrics: control,
treatmentMetrics: treatments,
primaryMetric: primaryComparisons[0] ?? this.emptyComparison(config.primaryMetric),
secondaryMetrics: secondaryComparisons,
guardrailViolations: violations,
runDate: new Date().toISOString(),
};
}
private compareMetric(
metricName: string,
control: ExperimentMetrics,
treatment: ExperimentMetrics,
minPracticalEffect: number
): MetricComparison {
let controlMean: number;
let treatmentMean: number;
let controlVariance: number;
let treatmentVariance: number;
if (metricName === 'conversion_rate') {
controlMean = control.conversionRate;
treatmentMean = treatment.conversionRate;
controlVariance = control.conversionVariance;
treatmentVariance = treatment.conversionVariance;
} else if (metricName === 'revenue_per_user') {
controlMean = control.revenuePerUser;
treatmentMean = treatment.revenuePerUser;
controlVariance = control.revenueVariance;
treatmentVariance = treatment.revenueVariance;
} else {
throw new Error(`Unknown metric: ${metricName}`);
}
// Welch's t-test
const seDiff = Math.sqrt(
controlVariance / control.users + treatmentVariance / treatment.users
);
const tStat = seDiff > 0 ? (treatmentMean - controlMean) / seDiff : 0;
// Approximate p-value (large sample, use normal distribution)
const pValue = 2 * (1 - normalCDF(Math.abs(tStat)));
// 95% CI for the difference
const diff = treatmentMean - controlMean;
const ci95: [number, number] = [diff - 1.96 * seDiff, diff + 1.96 * seDiff];
// Individual CIs
const controlSE = Math.sqrt(controlVariance / control.users);
const treatmentSE = Math.sqrt(treatmentVariance / treatment.users);
const relativeChange = controlMean !== 0 ? diff / controlMean : 0;
const practicallySignificant = Math.abs(relativeChange) >= minPracticalEffect;
return {
metricName,
control: {
mean: controlMean,
ci95: [controlMean - 1.96 * controlSE, controlMean + 1.96 * controlSE],
},
treatment: {
mean: treatmentMean,
ci95: [treatmentMean - 1.96 * treatmentSE, treatmentMean + 1.96 * treatmentSE],
},
absoluteChange: diff,
relativeChange,
pValue,
isSignificant: pValue < 0.05,
practicallySignificant,
};
}
private checkGuardrails(
guardrails: Array<{
metric: string;
maxDegradation: number;
severity: 'warning' | 'critical';
}>,
metrics: ExperimentMetrics[]
): GuardrailViolation[] {
const violations: GuardrailViolation[] = [];
const control = metrics.find((m) => m.variantId === 'control');
if (!control) return violations;
for (const treatment of metrics.filter((m) => m.variantId !== 'control')) {
for (const guardrail of guardrails) {
const controlValue = this.getMetricValue(guardrail.metric, control);
const treatmentValue = this.getMetricValue(guardrail.metric, treatment);
const relativeChange =
controlValue !== 0
? (treatmentValue - controlValue) / controlValue
: 0;
// Guardrails trigger on degradation (negative relative change beyond threshold)
if (relativeChange < -guardrail.maxDegradation) {
violations.push({
metricName: guardrail.metric,
controlValue,
treatmentValue,
relativeChange,
threshold: -guardrail.maxDegradation,
severity: guardrail.severity,
});
}
}
}
return violations;
}
private getMetricValue(metric: string, m: ExperimentMetrics): number {
const map: Record<string, number> = {
conversion_rate: m.conversionRate,
revenue_per_user: m.revenuePerUser,
};
return map[metric] ?? 0;
}
private emptyComparison(metricName: string): MetricComparison {
return {
metricName,
control: { mean: 0, ci95: [0, 0] },
treatment: { mean: 0, ci95: [0, 0] },
absoluteChange: 0,
relativeChange: 0,
pValue: 1,
isSignificant: false,
practicallySignificant: false,
};
}
private async fetchMetrics(
experimentId: string,
startDate: Date,
endDate: Date
): Promise<ExperimentMetrics[]> {
// Query ClickHouse (shown above)
const result = await this.clickhouse.query({
query: METRICS_QUERY,
query_params: {
experimentId,
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
},
format: 'JSONEachRow',
});
return result.json<ExperimentMetrics>();
}
}
const METRICS_QUERY = `/* SQL query from above */`;
CUPED Variance Reduction
CUPED (Controlled-experiment Using Pre-Experiment Data) reduces variance by removing the component explained by pre-experiment behavior:
// src/analysis/cuped.ts
interface CupedData {
userId: string;
variantId: string;
preMetric: number; // Metric value before experiment
postMetric: number; // Metric value during experiment
}
interface CupedResult {
adjustedMeans: Record<string, number>;
varianceReduction: number;
theta: number;
rho: number;
}
export function applyCuped(data: CupedData[]): CupedResult {
// Calculate theta (regression coefficient) using all data
const allPre = data.map((d) => d.preMetric);
const allPost = data.map((d) => d.postMetric);
const meanPre = mean(allPre);
const meanPost = mean(allPost);
const covPrePost =
data.reduce(
(s, d) => s + (d.preMetric - meanPre) * (d.postMetric - meanPost),
0
) / data.length;
const varPre =
data.reduce((s, d) => s + Math.pow(d.preMetric - meanPre, 2), 0) /
data.length;
const varPost =
data.reduce((s, d) => s + Math.pow(d.postMetric - meanPost, 2), 0) /
data.length;
const theta = varPre > 0 ? covPrePost / varPre : 0;
const rho = Math.sqrt(varPre * varPost) > 0 ? covPrePost / Math.sqrt(varPre * varPost) : 0;
// Apply CUPED adjustment per user
const adjusted = data.map((d) => ({
...d,
adjustedPost: d.postMetric - theta * (d.preMetric - meanPre),
}));
// Compute adjusted means per variant
const byVariant = new Map<string, number[]>();
for (const d of adjusted) {
const arr = byVariant.get(d.variantId) ?? [];
arr.push(d.adjustedPost);
byVariant.set(d.variantId, arr);
}
const adjustedMeans: Record<string, number> = {};
for (const [variantId, values] of byVariant.entries()) {
adjustedMeans[variantId] = mean(values);
}
// Variance reduction factor
const varianceReduction = 1 - Math.pow(rho, 2);
return { adjustedMeans, varianceReduction, theta, rho };
}
function mean(arr: number[]): number {
return arr.reduce((s, v) => s + v, 0) / arr.length;
}
Fetching Pre-Experiment Data
-- Get pre-experiment metric values (2-week lookback before experiment start)
SELECT
user_id,
variant_id, -- From experiment exposure
-- Pre-experiment metric (before experiment started)
countIf(
t.timestamp < {experimentStartDate: DateTime}
AND t.event = 'purchase_completed'
) AS pre_experiment_purchases,
-- Post-experiment metric
countIf(
t.timestamp >= {experimentStartDate: DateTime}
AND t.event = 'purchase_completed'
) AS post_experiment_purchases
FROM analytics.experiment_exposures e
LEFT JOIN tracking.events t
ON e.user_id = t.user_id
AND t.timestamp >= subtractDays({experimentStartDate: DateTime}, 14) -- 14-day pre window
AND t.timestamp <= {endDate: DateTime}
WHERE e.experiment_id = {experimentId: String}
GROUP BY e.user_id, e.variant_id
Segment Analysis
Segment analysis breaks down results by user subgroups to understand heterogeneous treatment effects.
// src/analysis/segment-analysis.ts
interface SegmentDefinition {
name: string;
attribute: string;
values: string[];
}
interface SegmentResult {
segment: string;
value: string;
controlUsers: number;
treatmentUsers: number;
controlConversionRate: number;
treatmentConversionRate: number;
relativeChange: number;
pValue: number;
isSignificant: boolean;
}
const DEFAULT_SEGMENTS: SegmentDefinition[] = [
{
name: 'device_type',
attribute: 'deviceType',
values: ['desktop', 'mobile', 'tablet'],
},
{
name: 'subscription_tier',
attribute: 'subscriptionTier',
values: ['free', 'pro', 'enterprise'],
},
{
name: 'new_vs_returning',
attribute: 'isNewUser',
values: ['true', 'false'],
},
];
export async function runSegmentAnalysis(
experimentId: string,
clickhouse: ClickHouseClient,
segments = DEFAULT_SEGMENTS
): Promise<SegmentResult[]> {
const results: SegmentResult[] = [];
for (const segment of segments) {
const query = `
WITH exposures AS (
SELECT
user_id,
variant_id,
JSONExtractString(properties, {attribute: String}) AS segment_value,
first_exposed_at
FROM analytics.experiment_exposures
WHERE experiment_id = {experimentId: String}
),
conversions AS (
SELECT DISTINCT user_id
FROM tracking.events
WHERE event = 'purchase_completed'
)
SELECT
e.variant_id,
e.segment_value,
count(DISTINCT e.user_id) AS users,
countIf(c.user_id IS NOT NULL) AS converted,
avg(c.user_id IS NOT NULL) AS conversion_rate
FROM exposures e
LEFT JOIN conversions c USING (user_id)
WHERE e.segment_value IN {values: Array(String)}
GROUP BY e.variant_id, e.segment_value
`;
const rows = await clickhouse.query({
query,
query_params: {
experimentId,
attribute: segment.attribute,
values: segment.values,
},
format: 'JSONEachRow',
}).then((r) => r.json<{
variant_id: string;
segment_value: string;
users: number;
converted: number;
conversion_rate: number;
}>());
// For each segment value, compare control vs treatment
for (const value of segment.values) {
const segRows = rows.filter((r) => r.segment_value === value);
const control = segRows.find((r) => r.variant_id === 'control');
const treatment = segRows.find((r) => r.variant_id !== 'control');
if (!control || !treatment || control.users < 50 || treatment.users < 50) {
continue;
}
const testResult = twoProportionZTest(
control.converted, control.users,
treatment.converted, treatment.users
);
results.push({
segment: segment.name,
value,
controlUsers: control.users,
treatmentUsers: treatment.users,
controlConversionRate: control.conversion_rate,
treatmentConversionRate: treatment.conversion_rate,
relativeChange: testResult.relativeChange,
pValue: testResult.pValue,
isSignificant: testResult.isSignificant,
});
}
}
return results;
}
Interpreting Segment Analysis
WARNING
Segment analysis is exploratory, not confirmatory. With 10 segments × 5 values = 50 tests, you expect ~2.5 false positives at by chance. Do not use segment analysis to "find" significance when the overall test was not significant.
Use segments to generate hypotheses for future targeted experiments.
Automated Analysis Reports
// src/analysis/report-generator.ts
interface ExperimentReport {
summary: {
winner: 'control' | 'treatment' | 'inconclusive' | 'guardrail-violated';
confidence: number;
primaryMetricLift: number;
recommendedAction: 'ship' | 'investigate' | 'stop' | 'extend';
};
statisticalResults: AnalysisResult;
segmentInsights: SegmentResult[];
sampleSizeAdequacy: {
current: number;
required: number;
percentComplete: number;
daysRemaining: number;
};
}
export function generateReport(
analysis: AnalysisResult,
segments: SegmentResult[],
config: { requiredSampleSize: number; dailyUsers: number }
): ExperimentReport {
const primaryLift = analysis.primaryMetric.relativeChange;
const isSignificant = analysis.status === 'significant';
const hasViolation = analysis.guardrailViolations.some(
(v) => v.severity === 'critical'
);
const winner: ExperimentReport['summary']['winner'] =
hasViolation ? 'guardrail-violated'
: !isSignificant ? 'inconclusive'
: primaryLift > 0 ? 'treatment'
: 'control';
const recommendedAction: ExperimentReport['summary']['recommendedAction'] =
hasViolation ? 'stop'
: winner === 'treatment' ? 'ship'
: winner === 'control' ? 'investigate'
: 'extend';
const totalUsers = analysis.controlMetrics.users +
analysis.treatmentMetrics.reduce((s, m) => s + m.users, 0);
const daysRemaining = Math.max(
0,
Math.ceil((config.requiredSampleSize - totalUsers) / config.dailyUsers)
);
return {
summary: {
winner,
confidence: isSignificant ? 1 - analysis.primaryMetric.pValue : 0,
primaryMetricLift: primaryLift,
recommendedAction,
},
statisticalResults: analysis,
segmentInsights: segments,
sampleSizeAdequacy: {
current: totalUsers,
required: config.requiredSampleSize,
percentComplete: Math.min(100, (totalUsers / config.requiredSampleSize) * 100),
daysRemaining,
},
};
}
Performance Optimization
Incremental Metric Computation
For experiments running for weeks, recomputing all metrics from scratch daily is expensive. Use incremental aggregation:
-- Incremental aggregation: only process yesterday's events
INSERT INTO analytics.experiment_daily_metrics
SELECT
experiment_id,
variant_id,
toDate(timestamp) AS date,
count(DISTINCT user_id) AS new_users,
countIf(event = 'purchase_completed') AS purchases,
sumIf(JSONExtractFloat(properties, 'revenue'), event = 'purchase_completed') AS revenue
FROM tracking.events
WHERE
toDate(timestamp) = yesterday()
GROUP BY experiment_id, variant_id, date;
-- Cumulative roll-up
SELECT
experiment_id,
variant_id,
sum(new_users) AS total_users,
sum(purchases) AS total_purchases,
sum(revenue) AS total_revenue
FROM analytics.experiment_daily_metrics
WHERE experiment_id = {experimentId: String}
GROUP BY experiment_id, variant_id;
Caching Analysis Results
// Cache analysis results for 1 hour (re-run nightly for accuracy)
const CACHE_TTL = 3600;
export async function getCachedAnalysis(
redis: Redis,
experimentId: string,
analyzer: ExperimentAnalyzer,
config: AnalysisConfig
): Promise<AnalysisResult> {
const cacheKey = `analysis:${experimentId}:${config.endDate.toISOString().split('T')[0]}`;
const cached = await redis.get(cacheKey);
if (cached) {
return JSON.parse(cached) as AnalysisResult;
}
const result = await analyzer.analyze(experimentId, config);
await redis.setex(cacheKey, CACHE_TTL, JSON.stringify(result));
return result;
}
War Story
The Segment That Wasn't Real
An analyst found a striking result: in the "power users" segment (top 10% by engagement), the treatment showed a 35% conversion lift with . The overall result was neutral ().
They shipped the treatment to power users only. Revenue didn't change.
Post-mortem: the power user segment was identified by the analyst using behavior data from the experiment period — data that was correlated with the treatment by definition (treatment improved engagement, which affected who was classified as a "power user"). This is endogeneity: using an outcome-influenced variable to define segments.
Correct approach: define segments using pre-experiment data only. Power users should be identified by their behavior in the 2 weeks before the experiment started.