google-site-verification: google61fe8ba583a51912.html
MACRO NEPAL
Multivariate Testing in Java: Advanced Experimentation Framework
awiskar acharya
November 18, 2025
8:51 am

Multivariate testing (MVT) allows testing multiple variables simultaneously to understand their individual and combined effects on user behavior. This is more sophisticated than A/B testing as it examines interactions between multiple factors.

Core Concepts

Key MVT Components

  • Factors: Independent variables being tested
  • Levels: Different variations of each factor
  • Treatment: Unique combination of factor levels
  • Response Variable: Metric being optimized
  • Interaction Effects: How factors influence each other

Implementation Framework

Dependencies

<!-- Statistics and Math --> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> <version>3.6.1</version> </dependency> <!-- JSON Processing --> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <version>2.15.0</version> </dependency> <!-- Caching --> <dependency> <groupId>com.github.ben-manes.caffeine</groupId> <artifactId>caffeine</artifactId> <version>3.1.6</version> </dependency> <!-- Database (Optional) --> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> </dependency>

Core MVT Implementation

Example 1: Experiment Definition and Management

// Experiment Status public enum ExperimentStatus { DRAFT, ACTIVE, PAUSED, COMPLETED, ARCHIVED } // Factor Definition @Data public class Factor { private final String name; private final String description; private final List<String> levels; private final FactorType type; private final Map<String, Object> metadata; public Factor(String name, String description, List<String> levels, FactorType type) { this.name = name; this.description = description; this.levels = new ArrayList<>(levels); this.type = type; this.metadata = new HashMap<>(); } public Factor withMetadata(String key, Object value) { this.metadata.put(key, value); return this; } public int getLevelIndex(String level) { return levels.indexOf(level); } public boolean isValidLevel(String level) { return levels.contains(level); } } // Factor Types public enum FactorType { CATEGORICAL, // Discrete categories ORDINAL, // Ordered categories NUMERICAL, // Continuous numerical values BINARY // Two levels (true/false) } // Treatment Definition @Data public class Treatment { private final String experimentId; private final Map<String, String> factorLevels; // factor -> level private final int treatmentId; private final double allocationPercentage; private final Map<String, Object> metadata; public Treatment(String experimentId, Map<String, String> factorLevels, int treatmentId, double allocationPercentage) { this.experimentId = experimentId; this.factorLevels = new HashMap<>(factorLevels); this.treatmentId = treatmentId; this.allocationPercentage = allocationPercentage; this.metadata = new HashMap<>(); } public Treatment withMetadata(String key, Object value) { this.metadata.put(key, value); return this; } public String getFactorLevel(String factorName) { return factorLevels.get(factorName); } public boolean hasFactor(String factorName) { return factorLevels.containsKey(factorName); } } // Experiment Definition @Data public class Experiment { private final String id; private final String name; private final String description; private final List<Factor> factors; private final List<Treatment> treatments; private final List<String> metrics; private final ExperimentStatus status; private final Instant startDate; private final Instant endDate; private final TargetingRule targetingRule; private final Map<String, Object> configuration; public Experiment(String id, String name, String description, List<Factor> factors, List<Treatment> treatments, List<String> metrics) { this.id = id; this.name = name; this.description = description; this.factors = new ArrayList<>(factors); this.treatments = new ArrayList<>(treatments); this.metrics = new ArrayList<>(metrics); this.status = ExperimentStatus.DRAFT; this.startDate = Instant.now(); this.endDate = startDate.plus(30, ChronoUnit.DAYS); // Default 30 days this.targetingRule = TargetingRule.ALL_USERS; this.configuration = new HashMap<>(); } public Experiment withConfiguration(String key, Object value) { this.configuration.put(key, value); return this; } public Experiment withTargetingRule(TargetingRule targetingRule) { this.targetingRule = targetingRule; return this; } public Experiment withDates(Instant startDate, Instant endDate) { this.startDate = startDate; this.endDate = endDate; return this; } public Factor getFactor(String factorName) { return factors.stream() .filter(f -> f.getName().equals(factorName)) .findFirst() .orElse(null); } public Treatment getTreatment(int treatmentId) { return treatments.stream() .filter(t -> t.getTreatmentId() == treatmentId) .findFirst() .orElse(null); } public boolean isActive() { return status == ExperimentStatus.ACTIVE && Instant.now().isAfter(startDate) && Instant.now().isBefore(endDate); } } // Targeting Rules public interface TargetingRule { TargetingRule ALL_USERS = (user, context) -> true; TargetingRule NO_USERS = (user, context) -> false; boolean isEligible(User user, Map<String, Object> context); static TargetingRule userSegment(String segment) { return (user, context) -> user != null && segment.equals(user.getSegment()); } static TargetingRule percentage(double percentage) { return (user, context) -> { if (user == null) return false; double hash = Math.abs((user.getId().hashCode() % 10000) / 10000.0); return hash < percentage; }; } static TargetingRule and(TargetingRule... rules) { return (user, context) -> { for (TargetingRule rule : rules) { if (!rule.isEligible(user, context)) return false; } return true; }; } static TargetingRule or(TargetingRule... rules) { return (user, context) -> { for (TargetingRule rule : rules) { if (rule.isEligible(user, context)) return true; } return false; }; } } // User Context @Data public class User { private final String id; private final String segment; private final Map<String, Object> attributes; public User(String id) { this.id = id; this.segment = "default"; this.attributes = new HashMap<>(); } public User withAttribute(String key, Object value) { this.attributes.put(key, value); return this; } public User withSegment(String segment) { this.segment = segment; return this; } }

Example 2: Treatment Assignment Engine

@Service @Slf4j public class TreatmentAssignmentService { private final ExperimentRepository experimentRepository; private final AssignmentCache assignmentCache; private final Random random; public TreatmentAssignmentService(ExperimentRepository experimentRepository) { this.experimentRepository = experimentRepository; this.assignmentCache = new AssignmentCache(); this.random = new Random(); } public AssignmentResult assignTreatment(String experimentId, User user, Map<String, Object> context) { try { Experiment experiment = experimentRepository.findById(experimentId); if (experiment == null) { return AssignmentResult.error("Experiment not found: " + experimentId); } if (!experiment.isActive()) { return AssignmentResult.error("Experiment is not active: " + experimentId); } // Check targeting if (!experiment.getTargetingRule().isEligible(user, context)) { return AssignmentResult.notEligible(); } // Check cache for existing assignment Assignment cachedAssignment = assignmentCache.get(experimentId, user.getId()); if (cachedAssignment != null) { return AssignmentResult.success(cachedAssignment); } // Assign treatment using hash-based consistent assignment Treatment treatment = assignTreatmentConsistently(experiment, user); Assignment assignment = new Assignment(experiment, user, treatment, context); assignmentCache.put(assignment); log.debug("Assigned treatment {} to user {} for experiment {}", treatment.getTreatmentId(), user.getId(), experimentId); return AssignmentResult.success(assignment); } catch (Exception e) { log.error("Failed to assign treatment for experiment: {}", experimentId, e); return AssignmentResult.error("Assignment failed: " + e.getMessage()); } } private Treatment assignTreatmentConsistently(Experiment experiment, User user) { // Use consistent hashing for stable assignments int hash = Math.abs((experiment.getId() + ":" + user.getId()).hashCode()); double randomValue = (hash % 10000) / 10000.0; double cumulativePercentage = 0.0; for (Treatment treatment : experiment.getTreatments()) { cumulativePercentage += treatment.getAllocationPercentage(); if (randomValue <= cumulativePercentage) { return treatment; } } // Fallback to first treatment return experiment.getTreatments().get(0); } public AssignmentResult assignMultipleTreatments(List<String> experimentIds, User user, Map<String, Object> context) { Map<String, Assignment> assignments = new HashMap<>(); List<String> errors = new ArrayList<>(); for (String experimentId : experimentIds) { AssignmentResult result = assignTreatment(experimentId, user, context); if (result.isSuccess()) { assignments.put(experimentId, result.getAssignment()); } else if (!result.isNotEligible()) { errors.add(experimentId + ": " + result.getErrorMessage()); } } if (!errors.isEmpty()) { return AssignmentResult.partialSuccess(assignments, errors); } return AssignmentResult.success(assignments); } public void clearAssignment(String experimentId, String userId) { assignmentCache.remove(experimentId, userId); } // Assignment result classes @Data public static class AssignmentResult { private final boolean success; private final Assignment assignment; private final Map<String, Assignment> assignments; private final String errorMessage; private final boolean notEligible; private final List<String> partialErrors; public static AssignmentResult success(Assignment assignment) { return new AssignmentResult(true, assignment, null, null, false, null); } public static AssignmentResult success(Map<String, Assignment> assignments) { return new AssignmentResult(true, null, assignments, null, false, null); } public static AssignmentResult error(String errorMessage) { return new AssignmentResult(false, null, null, errorMessage, false, null); } public static AssignmentResult notEligible() { return new AssignmentResult(false, null, null, null, true, null); } public static AssignmentResult partialSuccess(Map<String, Assignment> assignments, List<String> errors) { return new AssignmentResult(true, null, assignments, null, false, errors); } public boolean isPartialSuccess() { return success && partialErrors != null && !partialErrors.isEmpty(); } } @Data public static class Assignment { private final String assignmentId; private final Experiment experiment; private final User user; private final Treatment treatment; private final Map<String, Object> context; private final Instant assignedAt; public Assignment(Experiment experiment, User user, Treatment treatment, Map<String, Object> context) { this.assignmentId = UUID.randomUUID().toString(); this.experiment = experiment; this.user = user; this.treatment = treatment; this.context = new HashMap<>(context); this.assignedAt = Instant.now(); } public String getFactorLevel(String factorName) { return treatment.getFactorLevel(factorName); } public Map<String, String> getAllFactorLevels() { return new HashMap<>(treatment.getFactorLevels()); } } // Assignment cache for consistent user experience private static class AssignmentCache { private final Cache<String, Assignment> cache; public AssignmentCache() { this.cache = Caffeine.newBuilder() .maximumSize(100000) .expireAfterWrite(7, TimeUnit.DAYS) .build(); } public Assignment get(String experimentId, String userId) { String key = generateKey(experimentId, userId); return cache.getIfPresent(key); } public void put(Assignment assignment) { String key = generateKey(assignment.getExperiment().getId(), assignment.getUser().getId()); cache.put(key, assignment); } public void remove(String experimentId, String userId) { String key = generateKey(experimentId, userId); cache.invalidate(key); } private String generateKey(String experimentId, String userId) { return experimentId + ":" + userId; } } }

Example 3: Event Tracking and Data Collection

@Service @Slf4j public class EventTrackingService { private final EventRepository eventRepository; private final ExperimentRepository experimentRepository; private final ObjectMapper objectMapper; public EventTrackingService(EventRepository eventRepository, ExperimentRepository experimentRepository, ObjectMapper objectMapper) { this.eventRepository = eventRepository; this.experimentRepository = experimentRepository; this.objectMapper = objectMapper; } public void trackEvent(Event event) { try { // Validate event if (!isValidEvent(event)) { log.warn("Invalid event received: {}", event); return; } // Enrich event with experiment context if applicable enrichEventWithExperimentData(event); // Store event eventRepository.save(event); log.debug("Tracked event: {} for user {}", event.getEventType(), event.getUserId()); } catch (Exception e) { log.error("Failed to track event: {}", event, e); } } public void trackConversion(String experimentId, String userId, String conversionType, double value, Map<String, Object> properties) { Event event = new Event.Builder() .eventType("conversion") .userId(userId) .experimentId(experimentId) .conversionType(conversionType) .value(value) .properties(properties) .timestamp(Instant.now()) .build(); trackEvent(event); } public void trackExposure(String experimentId, String userId, int treatmentId, Map<String, Object> context) { Event event = new Event.Builder() .eventType("exposure") .userId(userId) .experimentId(experimentId) .treatmentId(treatmentId) .properties(context) .timestamp(Instant.now()) .build(); trackEvent(event); } public void trackCustomEvent(String eventType, String userId, Map<String, Object> properties) { Event event = new Event.Builder() .eventType(eventType) .userId(userId) .properties(properties) .timestamp(Instant.now()) .build(); trackEvent(event); } private boolean isValidEvent(Event event) { return event != null && event.getEventType() != null && event.getUserId() != null && event.getTimestamp() != null; } private void enrichEventWithExperimentData(Event event) { if (event.getExperimentId() == null) return; try { Experiment experiment = experimentRepository.findById(event.getExperimentId()); if (experiment != null && experiment.isActive()) { event.setExperimentName(experiment.getName()); event.setExperimentFactors(new ArrayList<>(experiment.getFactors())); } } catch (Exception e) { log.warn("Failed to enrich event with experiment data", e); } } public ExperimentData collectExperimentData(String experimentId, Instant startTime, Instant endTime) { try { Experiment experiment = experimentRepository.findById(experimentId); if (experiment == null) { throw new IllegalArgumentException("Experiment not found: " + experimentId); } List<Event> exposures = eventRepository.findExposures(experimentId, startTime, endTime); List<Event> conversions = eventRepository.findConversions(experimentId, startTime, endTime); return aggregateExperimentData(experiment, exposures, conversions); } catch (Exception e) { log.error("Failed to collect experiment data for: {}", experimentId, e); throw new RuntimeException("Data collection failed", e); } } private ExperimentData aggregateExperimentData(Experiment experiment, List<Event> exposures, List<Event> conversions) { ExperimentData data = new ExperimentData(experiment); // Group exposures by treatment Map<Integer, List<Event>> exposuresByTreatment = exposures.stream() .collect(Collectors.groupingBy(Event::getTreatmentId)); // Group conversions by treatment Map<Integer, List<Event>> conversionsByTreatment = conversions.stream() .collect(Collectors.groupingBy(Event::getTreatmentId)); // Calculate metrics for each treatment for (Treatment treatment : experiment.getTreatments()) { int treatmentId = treatment.getTreatmentId(); List<Event> treatmentExposures = exposuresByTreatment.getOrDefault(treatmentId, List.of()); List<Event> treatmentConversions = conversionsByTreatment.getOrDefault(treatmentId, List.of()); TreatmentData treatmentData = calculateTreatmentData( treatment, treatmentExposures, treatmentConversions); data.addTreatmentData(treatmentData); } // Calculate overall experiment metrics calculateOverallMetrics(data); return data; } private TreatmentData calculateTreatmentData(Treatment treatment, List<Event> exposures, List<Event> conversions) { TreatmentData data = new TreatmentData(treatment); data.setExposures(exposures.size()); data.setConversions(conversions.size()); // Calculate conversion rate double conversionRate = exposures.size() > 0 ? (double) conversions.size() / exposures.size() : 0.0; data.setConversionRate(conversionRate); // Calculate average conversion value double avgValue = conversions.stream() .mapToDouble(Event::getValue) .average() .orElse(0.0); data.setAverageValue(avgValue); // Calculate total value double totalValue = conversions.stream() .mapToDouble(Event::getValue) .sum(); data.setTotalValue(totalValue); return data; } private void calculateOverallMetrics(ExperimentData data) { long totalExposures = data.getTreatmentData().values().stream() .mapToLong(TreatmentData::getExposures) .sum(); long totalConversions = data.getTreatmentData().values().stream() .mapToLong(TreatmentData::getConversions) .sum(); data.setTotalExposures(totalExposures); data.setTotalConversions(totalConversions); data.setOverallConversionRate(totalExposures > 0 ? (double) totalConversions / totalExposures : 0.0); } // Data classes for experiment results @Data public static class ExperimentData { private final Experiment experiment; private final Map<Integer, TreatmentData> treatmentData; private long totalExposures; private long totalConversions; private double overallConversionRate; private Instant calculatedAt; public ExperimentData(Experiment experiment) { this.experiment = experiment; this.treatmentData = new HashMap<>(); this.calculatedAt = Instant.now(); } public void addTreatmentData(TreatmentData data) { treatmentData.put(data.getTreatment().getTreatmentId(), data); } public TreatmentData getTreatmentData(int treatmentId) { return treatmentData.get(treatmentId); } } @Data public static class TreatmentData { private final Treatment treatment; private long exposures; private long conversions; private double conversionRate; private double averageValue; private double totalValue; public TreatmentData(Treatment treatment) { this.treatment = treatment; } } // Event entity @Entity @Table(name = "experiment_events") @Data public static class Event { @Id private String id; private String eventType; private String userId; private String experimentId; private String experimentName; private Integer treatmentId; private String conversionType; private Double value; @Column(columnDefinition = "JSON") private String propertiesJson; private Instant timestamp; private Instant createdAt; @Transient private Map<String, Object> properties; @Transient private List<Factor> experimentFactors; public Event() { this.id = UUID.randomUUID().toString(); this.createdAt = Instant.now(); } @PostLoad public void deserializeProperties() { if (propertiesJson != null) { try { this.properties = objectMapper.readValue(propertiesJson, new TypeReference<Map<String, Object>>() {}); } catch (Exception e) { log.warn("Failed to deserialize event properties", e); } } } @PrePersist public void serializeProperties() { if (properties != null) { try { this.propertiesJson = objectMapper.writeValueAsString(properties); } catch (Exception e) { log.warn("Failed to serialize event properties", e); } } } public static class Builder { private final Event event; public Builder() { this.event = new Event(); } public Builder eventType(String eventType) { event.eventType = eventType; return this; } public Builder userId(String userId) { event.userId = userId; return this; } public Builder experimentId(String experimentId) { event.experimentId = experimentId; return this; } public Builder treatmentId(Integer treatmentId) { event.treatmentId = treatmentId; return this; } public Builder conversionType(String conversionType) { event.conversionType = conversionType; return this; } public Builder value(Double value) { event.value = value; return this; } public Builder properties(Map<String, Object> properties) { event.properties = properties != null ? new HashMap<>(properties) : new HashMap<>(); return this; } public Builder timestamp(Instant timestamp) { event.timestamp = timestamp; return this; } public Event build() { if (event.timestamp == null) { event.timestamp = Instant.now(); } return event; } } } }

Example 4: Statistical Analysis Engine

@Service @Slf4j public class StatisticalAnalysisService { private final EventTrackingService eventTrackingService; public StatisticalAnalysisService(EventTrackingService eventTrackingService) { this.eventTrackingService = eventTrackingService; } public AnalysisResult analyzeExperiment(String experimentId, Instant startTime, Instant endTime, AnalysisConfig config) { try { EventTrackingService.ExperimentData experimentData = eventTrackingService.collectExperimentData(experimentId, startTime, endTime); return performStatisticalAnalysis(experimentData, config); } catch (Exception e) { log.error("Failed to analyze experiment: {}", experimentId, e); throw new RuntimeException("Analysis failed", e); } } private AnalysisResult performStatisticalAnalysis( EventTrackingService.ExperimentData experimentData, AnalysisConfig config) { AnalysisResult result = new AnalysisResult(experimentData.getExperiment()); result.setAnalysisPeriodStart(experimentData.getCalculatedAt().minus(experimentData.getExperiment().getDuration())); result.setAnalysisPeriodEnd(experimentData.getCalculatedAt()); // Get control treatment (usually treatment 0) EventTrackingService.TreatmentData controlData = experimentData.getTreatmentData(0); if (controlData == null) { result.setError("Control treatment (0) not found"); return result; } // Compare each treatment against control for (EventTrackingService.TreatmentData treatmentData : experimentData.getTreatmentData().values()) { if (treatmentData.getTreatment().getTreatmentId() == 0) { continue; // Skip control in comparisons } TreatmentComparison comparison = compareTreatments( controlData, treatmentData, config); result.addComparison(comparison); } // Calculate experiment-wide statistics calculateExperimentStatistics(result, experimentData); return result; } private TreatmentComparison compareTreatments( EventTrackingService.TreatmentData control, EventTrackingService.TreatmentData variation, AnalysisConfig config) { TreatmentComparison comparison = new TreatmentComparison(control, variation); // Basic metrics comparison.setControlConversionRate(control.getConversionRate()); comparison.setVariationConversionRate(variation.getConversionRate()); // Calculate lift double lift = control.getConversionRate() > 0 ? (variation.getConversionRate() - control.getConversionRate()) / control.getConversionRate() * 100 : 0.0; comparison.setLiftPercentage(lift); // Perform statistical tests performConversionRateTest(comparison, config); performValueTest(comparison, config); return comparison; } private void performConversionRateTest(TreatmentComparison comparison, AnalysisConfig config) { try { EventTrackingService.TreatmentData control = comparison.getControl(); EventTrackingService.TreatmentData variation = comparison.getVariation(); long controlConversions = control.getConversions(); long controlExposures = control.getExposures(); long variationConversions = variation.getConversions(); long variationExposures = variation.getExposures(); // Chi-square test for conversion rates ChiSquareTest chiSquareTest = new ChiSquareTest(); long[][] contingencyTable = { {controlConversions, controlExposures - controlConversions}, {variationConversions, variationExposures - variationConversions} }; double pValue = chiSquareTest.chiSquareTest(contingencyTable); comparison.setConversionRatePValue(pValue); comparison.setConversionRateSignificant(pValue < config.getSignificanceLevel()); // Calculate confidence intervals double controlRate = control.getConversionRate(); double variationRate = variation.getConversionRate(); double controlSe = calculateStandardError(controlRate, controlExposures); double variationSe = calculateStandardError(variationRate, variationExposures); comparison.setControlConversionRateCI(calculateConfidenceInterval( controlRate, controlSe, config.getConfidenceLevel())); comparison.setVariationConversionRateCI(calculateConfidenceInterval( variationRate, variationSe, config.getConfidenceLevel())); } catch (Exception e) { log.warn("Failed to perform conversion rate test", e); comparison.setConversionRatePValue(Double.NaN); } } private void performValueTest(TreatmentComparison comparison, AnalysisConfig config) { try { // For value metrics, we'd need individual conversion values // This is a simplified implementation EventTrackingService.TreatmentData control = comparison.getControl(); EventTrackingService.TreatmentData variation = comparison.getVariation(); // T-test for average values (simplified) if (control.getConversions() > 30 && variation.getConversions() > 30) { TTest tTest = new TTest(); // In practice, you'd have individual conversion values // For now, we'll use a simplified approach double controlMean = control.getAverageValue(); double variationMean = variation.getAverageValue(); // Estimate standard deviations (simplified) double controlStd = controlMean * 0.5; // Rough estimate double variationStd = variationMean * 0.5; // Rough estimate double tStatistic = Math.abs(controlMean - variationMean) / Math.sqrt(controlStd * controlStd / control.getConversions() + variationStd * variationStd / variation.getConversions()); double pValue = 2 * (1 - new NormalDistribution().cumulativeProbability(tStatistic)); comparison.setValuePValue(pValue); comparison.setValueSignificant(pValue < config.getSignificanceLevel()); } } catch (Exception e) { log.warn("Failed to perform value test", e); comparison.setValuePValue(Double.NaN); } } private double calculateStandardError(double proportion, long sampleSize) { return Math.sqrt(proportion * (1 - proportion) / sampleSize); } private ConfidenceInterval calculateConfidenceInterval(double mean, double standardError, double confidenceLevel) { double zScore = getZScore(confidenceLevel); double marginOfError = zScore * standardError; return new ConfidenceInterval(mean - marginOfError, mean + marginOfError, confidenceLevel); } private double getZScore(double confidenceLevel) { // Common z-scores for confidence intervals Map<Double, Double> zScores = Map.of( 0.90, 1.645, 0.95, 1.960, 0.99, 2.576 ); return zScores.getOrDefault(confidenceLevel, 1.960); // Default to 95% } private void calculateExperimentStatistics(AnalysisResult result, EventTrackingService.ExperimentData data) { result.setTotalExposures(data.getTotalExposures()); result.setTotalConversions(data.getTotalConversions()); result.setOverallConversionRate(data.getOverallConversionRate()); // Calculate statistical power result.setStatisticalPower(calculateStatisticalPower(result)); // Check if experiment has reached sample size result.setSampleSizeReached(checkSampleSizeRequirement(result)); } private double calculateStatisticalPower(AnalysisResult result) { // Simplified power calculation // In practice, this would use more sophisticated methods long minSampleSize = result.getComparisons().stream() .mapToLong(c -> Math.min(c.getControl().getExposures(), c.getVariation().getExposures())) .min() .orElse(0); if (minSampleSize < 100) return 0.3; if (minSampleSize < 500) return 0.5; if (minSampleSize < 1000) return 0.7; if (minSampleSize < 5000) return 0.8; return 0.9; } private boolean checkSampleSizeRequirement(AnalysisResult result) { // Check if we have sufficient sample size for each comparison return result.getComparisons().stream() .allMatch(comparison -> { long controlExposures = comparison.getControl().getExposures(); long variationExposures = comparison.getVariation().getExposures(); return controlExposures >= 1000 && variationExposures >= 1000; }); } // Analysis configuration @Data public static class AnalysisConfig { private double significanceLevel = 0.05; private double confidenceLevel = 0.95; private boolean adjustForMultipleComparisons = true; private List<String> primaryMetrics = List.of("conversion_rate"); } // Analysis results @Data public static class AnalysisResult { private final Experiment experiment; private final List<TreatmentComparison> comparisons; private Instant analysisPeriodStart; private Instant analysisPeriodEnd; private long totalExposures; private long totalConversions; private double overallConversionRate; private double statisticalPower; private boolean sampleSizeReached; private String error; public AnalysisResult(Experiment experiment) { this.experiment = experiment; this.comparisons = new ArrayList<>(); } public void addComparison(TreatmentComparison comparison) { comparisons.add(comparison); } public boolean hasSignificantResults() { return comparisons.stream() .anyMatch(TreatmentComparison::isAnySignificant); } public List<TreatmentComparison> getSignificantComparisons() { return comparisons.stream() .filter(TreatmentComparison::isAnySignificant) .collect(Collectors.toList()); } } @Data public static class TreatmentComparison { private final EventTrackingService.TreatmentData control; private final EventTrackingService.TreatmentData variation; private double controlConversionRate; private double variationConversionRate; private double liftPercentage; private double conversionRatePValue; private boolean conversionRateSignificant; private ConfidenceInterval controlConversionRateCI; private ConfidenceInterval variationConversionRateCI; private double valuePValue; private boolean valueSignificant; public TreatmentComparison(EventTrackingService.TreatmentData control, EventTrackingService.TreatmentData variation) { this.control = control; this.variation = variation; } public boolean isAnySignificant() { return conversionRateSignificant || valueSignificant; } public String getRecommendation() { if (!conversionRateSignificant && !valueSignificant) { return "NO_DIFFERENCE"; } if (variationConversionRate > controlConversionRate) { return "IMPLEMENT_VARIATION"; } else { return "KEEP_CONTROL"; } } } @Data public static class ConfidenceInterval { private final double lowerBound; private final double upperBound; private final double confidenceLevel; public boolean contains(double value) { return value >= lowerBound && value <= upperBound; } public double getWidth() { return upperBound - lowerBound; } } }

Example 5: Full Factorial Design Generator

@Service @Slf4j public class ExperimentDesignService { public Experiment createFullFactorialDesign(String experimentId, String name, List<Factor> factors, List<String> metrics) { log.info("Creating full factorial design for experiment: {}", name); // Generate all possible treatment combinations List<Treatment> treatments = generateFullFactorialTreatments(experimentId, factors); // Calculate equal allocation percentages double allocationPercentage = 100.0 / treatments.size(); treatments.forEach(t -> t.setAllocationPercentage(allocationPercentage)); // Create experiment Experiment experiment = new Experiment(experimentId, name, "Full factorial design with " + treatments.size() + " treatments", factors, treatments, metrics); log.info("Created full factorial design with {} factors and {} treatments", factors.size(), treatments.size()); return experiment; } public Experiment createFractionalFactorialDesign(String experimentId, String name, List<Factor> factors, List<String> metrics, int resolution) { log.info("Creating fractional factorial design for experiment: {}", name); // Generate fractional factorial treatments List<Treatment> treatments = generateFractionalFactorialTreatments( experimentId, factors, resolution); // Calculate allocation percentages double allocationPercentage = 100.0 / treatments.size(); treatments.forEach(t -> t.setAllocationPercentage(allocationPercentage)); Experiment experiment = new Experiment(experimentId, name, "Fractional factorial design with " + treatments.size() + " treatments", factors, treatments, metrics); log.info("Created fractional factorial design with {} factors and {} treatments", factors.size(), treatments.size()); return experiment; } public Experiment createOptimalDesign(String experimentId, String name, List<Factor> factors, List<String> metrics, int maxTreatments) { log.info("Creating optimal design for experiment: {}", name); // Generate optimal design using D-optimality criterion List<Treatment> treatments = generateOptimalDesign(experimentId, factors, maxTreatments); double allocationPercentage = 100.0 / treatments.size(); treatments.forEach(t -> t.setAllocationPercentage(allocationPercentage)); Experiment experiment = new Experiment(experimentId, name, "Optimal design with " + treatments.size() + " treatments", factors, treatments, metrics); log.info("Created optimal design with {} factors and {} treatments", factors.size(), treatments.size()); return experiment; } private List<Treatment> generateFullFactorialTreatments(String experimentId, List<Factor> factors) { List<Map<String, String>> combinations = generateAllCombinations(factors); List<Treatment> treatments = new ArrayList<>(); for (int i = 0; i < combinations.size(); i++) { Map<String, String> factorLevels = combinations.get(i); Treatment treatment = new Treatment(experimentId, factorLevels, i, 0.0); treatments.add(treatment); } return treatments; } private List<Map<String, String>> generateAllCombinations(List<Factor> factors) { List<Map<String, String>> combinations = new ArrayList<>(); combinations.add(new HashMap<>()); for (Factor factor : factors) { List<Map<String, String>> newCombinations = new ArrayList<>(); for (Map<String, String> combination : combinations) { for (String level : factor.getLevels()) { Map<String, String> newCombination = new HashMap<>(combination); newCombination.put(factor.getName(), level); newCombinations.add(newCombination); } } combinations = newCombinations; } return combinations; } private List<Treatment> generateFractionalFactorialTreatments(String experimentId, List<Factor> factors, int resolution) { // Simplified fractional factorial implementation // In practice, this would use more sophisticated algorithms List<Map<String, String>> allCombinations = generateAllCombinations(factors); int totalCombinations = allCombinations.size(); int fractionSize = totalCombinations / (int) Math.pow(2, resolution - 1); // Select a fraction of all combinations List<Map<String, String>> selectedCombinations = allCombinations.stream() .filter(combination -> shouldIncludeInFraction(combination, factors, resolution)) .limit(fractionSize) .collect(Collectors.toList()); List<Treatment> treatments = new ArrayList<>(); for (int i = 0; i < selectedCombinations.size(); i++) { Map<String, String> factorLevels = selectedCombinations.get(i); Treatment treatment = new Treatment(experimentId, factorLevels, i, 0.0); treatments.add(treatment); } return treatments; } private boolean shouldIncludeInFraction(Map<String, String> combination, List<Factor> factors, int resolution) { // Simplified selection criteria // In practice, this would use defining contrasts int hash = combination.hashCode(); return hash % resolution == 0; } private List<Treatment> generateOptimalDesign(String experimentId, List<Factor> factors, int maxTreatments) { // Simplified optimal design implementation // In practice, this would use algorithms like Federov's algorithm List<Map<String, String>> allCombinations = generateAllCombinations(factors); // Select treatments that maximize information (simplified) List<Map<String, String>> selectedCombinations = allCombinations.stream() .sorted(Comparator.comparing(this::calculateDesignEfficiency)) .limit(maxTreatments) .collect(Collectors.toList()); List<Treatment> treatments = new ArrayList<>(); for (int i = 0; i < selectedCombinations.size(); i++) { Map<String, String> factorLevels = selectedCombinations.get(i); Treatment treatment = new Treatment(experimentId, factorLevels, i, 0.0); treatments.add(treatment); } return treatments; } private double calculateDesignEfficiency(Map<String, String> combination) { // Simplified efficiency calculation // In practice, this would calculate D-efficiency or other criteria return Math.random(); // Placeholder } public DesignEvaluation evaluateDesign(Experiment experiment) { DesignEvaluation evaluation = new DesignEvaluation(experiment); // Calculate design properties evaluation.setNumberOfTreatments(experiment.getTreatments().size()); evaluation.setNumberOfFactors(experiment.getFactors().size()); evaluation.setDesignEfficiency(calculateDesignEfficiency(experiment)); evaluation.setPowerAnalysis(estimateStatisticalPower(experiment)); evaluation.setRequiredSampleSize(estimateRequiredSampleSize(experiment)); return evaluation; } private double calculateDesignEfficiency(Experiment experiment) { // Simplified efficiency calculation int numFactors = experiment.getFactors().size(); int numTreatments = experiment.getTreatments().size(); // Full factorial would have 2^numFactors treatments int fullFactorialSize = (int) Math.pow(2, numFactors); if (numTreatments == fullFactorialSize) { return 1.0; // Full factorial is 100% efficient } else { return (double) numTreatments / fullFactorialSize; } } private PowerAnalysis estimateStatisticalPower(Experiment experiment) { PowerAnalysis analysis = new PowerAnalysis(); // Simplified power estimation int totalSampleSize = 1000; // Example double effectSize = 0.1; // Small effect double alpha = 0.05; // Simplified power calculation double power = calculatePower(totalSampleSize, effectSize, alpha); analysis.setEstimatedPower(power); analysis.setEffectSize(effectSize); analysis.setAlpha(alpha); analysis.setTotalSampleSize(totalSampleSize); return analysis; } private long estimateRequiredSampleSize(Experiment experiment) { // Simplified sample size calculation double power = 0.8; double alpha = 0.05; double effectSize = 0.1; // Using formula for two-sample proportion test double zAlpha = 1.96; // for alpha = 0.05 double zBeta = 0.84; // for power = 0.8 double p = 0.5; // assumed proportion double sampleSize = Math.pow(zAlpha + zBeta, 2) * p * (1 - p) * 2 / Math.pow(effectSize, 2); return (long) Math.ceil(sampleSize); } private double calculatePower(long sampleSize, double effectSize, double alpha) { // Simplified power calculation double basePower = 0.5; double sizeFactor = Math.log10(sampleSize) / 4.0; // Normalize double effectFactor = effectSize * 10; return Math.min(0.99, basePower + sizeFactor * effectFactor); } @Data public static class DesignEvaluation { private final Experiment experiment; private int numberOfTreatments; private int numberOfFactors; private double designEfficiency; private PowerAnalysis powerAnalysis; private long requiredSampleSize; private List<String> recommendations; public DesignEvaluation(Experiment experiment) { this.experiment = experiment; this.recommendations = new ArrayList<>(); } public void addRecommendation(String recommendation) { recommendations.add(recommendation); } public boolean isFeasible() { return designEfficiency > 0.5 && powerAnalysis.getEstimatedPower() > 0.7 && requiredSampleSize < 100000; // Arbitrary limit } } @Data public static class PowerAnalysis { private double estimatedPower; private double effectSize; private double alpha; private long totalSampleSize; } }

Spring Boot Integration

Example 6: REST Controller and Configuration

@RestController @RequestMapping("/api/experiments") @Slf4j public class ExperimentController { private final TreatmentAssignmentService assignmentService; private final EventTrackingService eventTrackingService; private final StatisticalAnalysisService analysisService; private final ExperimentDesignService designService; private final ExperimentRepository experimentRepository; public ExperimentController(TreatmentAssignmentService assignmentService, EventTrackingService eventTrackingService, StatisticalAnalysisService analysisService, ExperimentDesignService designService, ExperimentRepository experimentRepository) { this.assignmentService = assignmentService; this.eventTrackingService = eventTrackingService; this.analysisService = analysisService; this.designService = designService; this.experimentRepository = experimentRepository; } @PostMapping("/{experimentId}/assign") public ResponseEntity<AssignmentResponse> assignTreatment( @PathVariable String experimentId, @RequestBody AssignmentRequest request) { try { User user = new User(request.getUserId()) .withSegment(request.getSegment()) .withAttributes(request.getUserAttributes()); TreatmentAssignmentService.AssignmentResult result = assignmentService.assignTreatment(experimentId, user, request.getContext()); if (result.isSuccess()) { // Track exposure eventTrackingService.trackExposure( experimentId, request.getUserId(), result.getAssignment().getTreatment().getTreatmentId(), request.getContext() ); AssignmentResponse response = new AssignmentResponse(result.getAssignment()); return ResponseEntity.ok(response); } else if (result.isNotEligible()) { return ResponseEntity.status(HttpStatus.NOT_ACCEPTABLE) .body(AssignmentResponse.notEligible()); } else { return ResponseEntity.status(HttpStatus.BAD_REQUEST) .body(AssignmentResponse.error(result.getErrorMessage())); } } catch (Exception e) { log.error("Failed to assign treatment for experiment: {}", experimentId, e); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) .body(AssignmentResponse.error("Assignment failed")); } } @PostMapping("/{experimentId}/conversion") public ResponseEntity<ConversionResponse> trackConversion( @PathVariable String experimentId, @RequestBody ConversionRequest request) { try { eventTrackingService.trackConversion( experimentId, request.getUserId(), request.getConversionType(), request.getValue(), request.getProperties() ); return ResponseEntity.ok(new ConversionResponse(true, "Conversion tracked")); } catch (Exception e) { log.error("Failed to track conversion for experiment: {}", experimentId, e); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) .body(new ConversionResponse(false, "Tracking failed")); } } @GetMapping("/{experimentId}/analysis") public ResponseEntity<AnalysisResult> analyzeExperiment( @PathVariable String experimentId, @RequestParam(defaultValue = "7") int days) { try { Instant endTime = Instant.now(); Instant startTime = endTime.minus(days, ChronoUnit.DAYS); StatisticalAnalysisService.AnalysisResult result = analysisService.analyzeExperiment(experimentId, startTime, endTime, new StatisticalAnalysisService.AnalysisConfig()); return ResponseEntity.ok(result); } catch (Exception e) { log.error("Failed to analyze experiment: {}", experimentId, e); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } } @PostMapping("/design") public ResponseEntity<DesignResponse> createExperimentDesign( @RequestBody DesignRequest request) { try { ExperimentDesignService.DesignEvaluation evaluation = null; Experiment experiment = null; switch (request.getDesignType()) { case FULL_FACTORIAL: experiment = designService.createFullFactorialDesign( request.getExperimentId(), request.getName(), request.getFactors(), request.getMetrics() ); break; case FRACTIONAL_FACTORIAL: experiment = designService.createFractionalFactorialDesign( request.getExperimentId(), request.getName(), request.getFactors(), request.getMetrics(), request.getResolution() ); break; case OPTIMAL: experiment = designService.createOptimalDesign( request.getExperimentId(), request.getName(), request.getFactors(), request.getMetrics(), request.getMaxTreatments() ); break; } if (experiment != null) { evaluation = designService.evaluateDesign(experiment); experimentRepository.save(experiment); } DesignResponse response = new DesignResponse(experiment, evaluation); return ResponseEntity.ok(response); } catch (Exception e) { log.error("Failed to create experiment design", e); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) .body(DesignResponse.error("Design creation failed")); } } // Request/Response DTOs @Data public static class AssignmentRequest { private String userId; private String segment; private Map<String, Object> userAttributes = new HashMap<>(); private Map<String, Object> context = new HashMap<>(); } @Data public static class AssignmentResponse { private boolean success; private TreatmentAssignmentService.Assignment assignment; private String errorMessage; private boolean notEligible; public AssignmentResponse(TreatmentAssignmentService.Assignment assignment) { this.success = true; this.assignment = assignment; } public static AssignmentResponse error(String errorMessage) { AssignmentResponse response = new AssignmentResponse(); response.success = false; response.errorMessage = errorMessage; return response; } public static AssignmentResponse notEligible() { AssignmentResponse response = new AssignmentResponse(); response.success = false; response.notEligible = true; return response; } } @Data public static class ConversionRequest { private String userId; private String conversionType; private double value; private Map<String, Object> properties = new HashMap<>(); } @Data public static class ConversionResponse { private boolean success; private String message; public ConversionResponse(boolean success, String message) { this.success = success; this.message = message; } } @Data public static class DesignRequest { private String experimentId; private String name; private DesignType designType; private List<Factor> factors; private List<String> metrics; private Integer resolution; private Integer maxTreatments; } @Data public static class DesignResponse { private boolean success; private Experiment experiment; private ExperimentDesignService.DesignEvaluation evaluation; private String errorMessage; public DesignResponse(Experiment experiment, ExperimentDesignService.DesignEvaluation evaluation) { this.success = true; this.experiment = experiment; this.evaluation = evaluation; } public static DesignResponse error(String errorMessage) { DesignResponse response = new DesignResponse(); response.success = false; response.errorMessage = errorMessage; return response; } } public enum DesignType { FULL_FACTORIAL, FRACTIONAL_FACTORIAL, OPTIMAL } } @Configuration @EnableConfigurationProperties(MVTConfigurationProperties.class) @Slf4j public class MVTAutoConfiguration { @Bean @ConditionalOnMissingBean public TreatmentAssignmentService treatmentAssignmentService( ExperimentRepository experimentRepository) { return new TreatmentAssignmentService(experimentRepository); } @Bean @ConditionalOnMissingBean public EventTrackingService eventTrackingService( EventRepository eventRepository, ExperimentRepository experimentRepository, ObjectMapper objectMapper) { return new EventTrackingService(eventRepository, experimentRepository, objectMapper); } @Bean @ConditionalOnMissingBean public StatisticalAnalysisService statisticalAnalysisService( EventTrackingService eventTrackingService) { return new StatisticalAnalysisService(eventTrackingService); } @Bean @ConditionalOnMissingBean public ExperimentDesignService experimentDesignService() { return new ExperimentDesignService(); } @Bean public ExperimentController experimentController( TreatmentAssignmentService assignmentService, EventTrackingService eventTrackingService, StatisticalAnalysisService analysisService, ExperimentDesignService designService, ExperimentRepository experimentRepository) { return new ExperimentController(assignmentService, eventTrackingService, analysisService, designService, experimentRepository); } } @ConfigurationProperties(prefix = "mvt") @Data public class MVTConfigurationProperties { private boolean enabled = true; private Cache cache = new Cache(); private Tracking tracking = new Tracking(); private Analysis analysis = new Analysis(); @Data public static class Cache { private long maximumSize = 100000; private long expireAfterDays = 7; } @Data public static class Tracking { private boolean enabled = true; private String eventStore = "database"; // database, redis, kafka private int batchSize = 100; private Duration flushInterval = Duration.ofSeconds(30); } @Data public static class Analysis { private double significanceLevel = 0.05; private double confidenceLevel = 0.95; private Duration analysisInterval = Duration.ofHours(1); } }

Best Practices

Configuration Example

# application-mvt.yml mvt: enabled: true cache: maximum-size: 100000 expire-after-days: 7 tracking: enabled: true event-store: "database" batch-size: 100 flush-interval: 30s analysis: significance-level: 0.05 confidence-level: 0.95 analysis-interval: 1h # Example experiment definition experiments: homepage-redesign: id: "homepage-redesign-2024" name: "Homepage Redesign Multivariate Test" factors: - name: "header_design" type: CATEGORICAL levels: ["minimal", "standard", "enhanced"] - name: "cta_button" type: CATEGORICAL levels: ["primary", "secondary", "gradient"] - name: "social_proof" type: BINARY levels: ["enabled", "disabled"] metrics: - "signup_conversion_rate" - "time_on_page" - "bounce_rate" targeting: rule: "percentage(0.5)" # 50% of users design: type: "FRACTIONAL_FACTORIAL" resolution: 3

Conclusion

Multivariate testing in Java provides a powerful framework for understanding complex user interactions:

Key Implementation Patterns:

  1. Experiment Design: Full factorial, fractional factorial, and optimal designs
  2. Treatment Assignment: Consistent hashing for stable user experiences
  3. Event Tracking: Comprehensive data collection for analysis
  4. Statistical Analysis: Advanced hypothesis testing and confidence intervals
  5. Design Evaluation: Power analysis and sample size estimation

Benefits of MVT:

  • Interaction Insights: Understand how factors influence each other
  • Efficient Testing: Test multiple variables simultaneously
  • Optimal Combinations: Find the best combination of factors
  • Data-Driven Decisions: Make informed decisions based on statistical evidence

Best Practices:

  • Start with fractional factorial designs for many factors
  • Use consistent assignment to maintain user experience
  • Monitor statistical power and sample size requirements
  • Implement proper multiple comparison corrections
  • Validate assumptions and check for interaction effects

Multivariate testing enables organizations to optimize complex user experiences by systematically testing and understanding the impact of multiple variables and their interactions.

Category : JAVA

Leave a Reply

Your email address will not be published. Required fields are marked *


Macro Nepal Helper