Overview
Elastic APM (Application Performance Monitoring) is a distributed tracing system that helps monitor applications in real-time. The Java agent automatically instruments applications to collect performance metrics, errors, and distributed traces.
Key Features
- Automatic Instrumentation: No code changes required for basic setup
- Distributed Tracing: Track requests across microservices
- Performance Metrics: Response times, throughput, error rates
- Error Tracking: Capture and analyze exceptions
- OpenTelemetry Support: Compatible with OpenTelemetry standards
Installation and Setup
1. Download and Configuration
# Download the latest agent wget https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/1.36.0/elastic-apm-agent-1.36.0.jar # Basic configuration export ELASTIC_APM_SERVICE_NAME=my-java-service export ELASTIC_APM_APPLICATION_PACKAGES=com.mycompany export ELASTIC_APM_SERVER_URL=http://localhost:8200 export ELASTIC_APM_SECRET_TOKEN=your-secret-token # Start application with agent java -javaagent:./elastic-apm-agent-1.36.0.jar \ -jar my-application.jar
2. Configuration Options
# application.properties or environment variables # Required elastic.apm.service_name=my-java-service elastic.apm.application_packages=com.mycompany elastic.apm.server_url=http://localhost:8200 # Optional elastic.apm.secret_token=your-secret-token elastic.apm.environment=production elastic.apm.transaction_sample_rate=1.0 elastic.apm.capture_body=all elastic.apm.capture_headers=true elastic.apm.log_level=INFO elastic.apm.ignore_urls=/health,/metrics
Manual Instrumentation
1. Basic Transaction and Span Creation
import co.elastic.apm.api.ElasticApm;
import co.elastic.apm.api.Transaction;
import co.elastic.apm.api.Span;
import co.elastic.apm.api.CaptureTransaction;
public class OrderService {
@CaptureTransaction(type = "Service", value = "processOrder")
public void processOrder(Order order) {
try {
// This method will be automatically traced
validateOrder(order);
processPayment(order);
updateInventory(order);
sendConfirmation(order);
} catch (Exception e) {
ElasticApm.currentTransaction().captureException(e);
throw e;
}
}
// Manual transaction creation
public void manualTransactionExample(Order order) {
Transaction transaction = ElasticApm.startTransaction();
try {
transaction.setName("Manual Order Processing");
transaction.setType(Transaction.TYPE_REQUEST);
// Add custom context
transaction.addLabel("orderId", order.getId());
transaction.addLabel("customerId", order.getCustomerId());
transaction.addLabel("amount", order.getTotalAmount());
processOrderLogic(order);
transaction.setResult("success");
} catch (Exception e) {
transaction.captureException(e);
transaction.setResult("failure");
throw e;
} finally {
transaction.end();
}
}
// Nested spans example
public void processOrderWithSpans(Order order) {
Transaction transaction = ElasticApm.currentTransaction();
try (Span validationSpan = transaction.startSpan("validation", "business", "order-validation")) {
validationSpan.setName("Validate Order");
validateOrder(order);
}
try (Span paymentSpan = transaction.startSpan("payment", "external", "payment-gateway")) {
paymentSpan.setName("Process Payment");
processPayment(order);
}
try (Span inventorySpan = transaction.startSpan("inventory", "database", "inventory-update")) {
inventorySpan.setName("Update Inventory");
updateInventory(order);
}
}
private void validateOrder(Order order) {
// Validation logic
if (order.getItems().isEmpty()) {
throw new IllegalArgumentException("Order must have items");
}
}
private void processPayment(Order order) {
// Payment processing logic
}
private void updateInventory(Order order) {
// Inventory update logic
}
private void sendConfirmation(Order order) {
// Email sending logic
}
}
2. Custom Span Creation
import co.elastic.apm.api.Span;
import co.elastic.apm.api.Transaction;
public class DatabaseService {
public User findUserById(String userId) {
Span span = ElasticApm.currentSpan();
try {
span.setName("Database Query: findUserById");
span.setType("db");
span.setSubtype("mysql");
span.addLabel("db.instance", "users_db");
span.addLabel("db.statement", "SELECT * FROM users WHERE id = ?");
span.addLabel("db.user_id", userId);
// Simulate database call
Thread.sleep(50);
return new User(userId, "John Doe");
} catch (Exception e) {
span.captureException(e);
throw new RuntimeException("Database error", e);
}
}
public void batchProcessUsers(List<String> userIds) {
Transaction transaction = ElasticApm.currentTransaction();
for (String userId : userIds) {
try (Span userSpan = transaction.startSpan("processing", "business", "user-processing")) {
userSpan.setName("Process User: " + userId);
userSpan.addLabel("user.id", userId);
processSingleUser(userId);
}
}
}
private void processSingleUser(String userId) {
// Process individual user
}
}
class User {
private String id;
private String name;
public User(String id, String name) {
this.id = id;
this.name = name;
}
// getters and setters
}
Spring Boot Integration
1. Spring Boot Configuration
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import co.elastic.apm.api.ElasticApm;
@Configuration
public class ApmConfiguration implements WebMvcConfigurer {
// Custom interceptor for additional context
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(new ApmInterceptor());
}
}
// Custom interceptor
import org.springframework.web.servlet.HandlerInterceptor;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
public class ApmInterceptor implements HandlerInterceptor {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Transaction transaction = ElasticApm.currentTransaction();
// Add custom context from request
transaction.addLabel("user.agent", request.getHeader("User-Agent"));
transaction.addLabel("client.ip", getClientIp(request));
transaction.addLabel("session.id", request.getSession().getId());
return true;
}
private String getClientIp(HttpServletRequest request) {
String xfHeader = request.getHeader("X-Forwarded-For");
if (xfHeader != null) {
return xfHeader.split(",")[0];
}
return request.getRemoteAddr();
}
}
2. Spring Service Instrumentation
import org.springframework.stereotype.Service;
import org.springframework.beans.factory.annotation.Autowired;
import co.elastic.apm.api.ElasticApm;
import co.elastic.apm.api.Transaction;
import co.elastic.apm.api.CaptureTransaction;
@Service
public class PaymentService {
@Autowired
private PaymentGateway paymentGateway;
@Autowired
private NotificationService notificationService;
@CaptureTransaction(type = "Service", value = "processPayment")
public PaymentResult processPayment(PaymentRequest request) {
Transaction transaction = ElasticApm.currentTransaction();
try {
// Add business context
transaction.addLabel("payment.amount", request.getAmount());
transaction.addLabel("payment.currency", request.getCurrency());
transaction.addLabel("payment.method", request.getPaymentMethod());
transaction.addLabel("customer.id", request.getCustomerId());
// Process payment
PaymentGatewayResponse response = paymentGateway.charge(request);
// Send notification
notificationService.sendPaymentConfirmation(request, response);
return PaymentResult.success(response.getTransactionId());
} catch (PaymentException e) {
transaction.captureException(e);
transaction.addLabel("payment.error_code", e.getErrorCode());
return PaymentResult.failure(e.getMessage());
}
}
}
@Service
public class NotificationService {
@CaptureTransaction(type = "Messaging", value = "sendNotification")
public void sendPaymentConfirmation(PaymentRequest request, PaymentGatewayResponse response) {
Transaction transaction = ElasticApm.currentTransaction();
try {
transaction.addLabel("notification.type", "payment_confirmation");
transaction.addLabel("notification.recipient", request.getCustomerEmail());
// Simulate sending notification
Thread.sleep(100);
} catch (Exception e) {
transaction.captureException(e);
// Don't throw - notification failure shouldn't fail payment
}
}
}
Database and HTTP Client Instrumentation
1. HTTP Client Instrumentation
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import co.elastic.apm.api.ElasticApm;
import co.elastic.apm.api.Span;
@Service
public class ExternalServiceClient {
@Autowired
private RestTemplate restTemplate;
public UserProfile fetchUserProfile(String userId) {
Span span = ElasticApm.currentSpan();
try {
span.setName("HTTP Request: User Service");
span.setType("external");
span.setSubtype("http");
String url = "https://user-service/api/users/" + userId;
// Add headers for distributed tracing
HttpHeaders headers = createHeadersWithTracing();
ResponseEntity<UserProfile> response = restTemplate.exchange(
url, HttpMethod.GET, new HttpEntity<>(headers), UserProfile.class);
span.addLabel("http.url", url);
span.addLabel("http.status_code", response.getStatusCodeValue());
span.addLabel("http.method", "GET");
return response.getBody();
} catch (Exception e) {
span.captureException(e);
span.addLabel("http.error", e.getMessage());
throw new RuntimeException("Failed to fetch user profile", e);
}
}
private HttpHeaders createHeadersWithTracing() {
HttpHeaders headers = new HttpHeaders();
headers.set("Content-Type", "application/json");
// Inject tracing headers for distributed tracing
ElasticApm.currentTransaction().injectTraceHeaders(headers::set);
return headers;
}
public void batchUpdateUsers(List<User> users) {
Transaction transaction = ElasticApm.currentTransaction();
for (User user : users) {
try (Span userSpan = transaction.startSpan("external", "http", "user-update")) {
userSpan.setName("Update User: " + user.getId());
String url = "https://user-service/api/users/" + user.getId();
restTemplate.put(url, user);
userSpan.addLabel("http.url", url);
userSpan.addLabel("http.method", "PUT");
}
}
}
}
2. Database Instrumentation
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import co.elastic.apm.api.ElasticApm;
import co.elastic.apm.api.Span;
@Repository
public class UserRepository {
@Autowired
private JdbcTemplate jdbcTemplate;
public User findById(String userId) {
Span span = ElasticApm.currentSpan();
try {
span.setName("Database Query: findUserById");
span.setType("db");
span.setSubtype("postgresql");
String sql = "SELECT id, name, email FROM users WHERE id = ?";
span.addLabel("db.instance", "user_database");
span.addLabel("db.statement", sql);
span.addLabel("db.user.id", userId);
return jdbcTemplate.queryForObject(sql, new Object[]{userId}, (rs, rowNum) ->
new User(rs.getString("id"), rs.getString("name"), rs.getString("email")));
} catch (Exception e) {
span.captureException(e);
span.addLabel("db.error", e.getMessage());
throw new RuntimeException("Database query failed", e);
}
}
public void batchInsertUsers(List<User> users) {
Transaction transaction = ElasticApm.currentTransaction();
String sql = "INSERT INTO users (id, name, email) VALUES (?, ?, ?)";
try (Span batchSpan = transaction.startSpan("db", "postgresql", "batch-insert")) {
batchSpan.setName("Batch Insert Users");
batchSpan.addLabel("db.statement", sql);
batchSpan.addLabel("db.rows.affected", users.size());
jdbcTemplate.batchUpdate(sql, users, users.size(), (ps, user) -> {
ps.setString(1, user.getId());
ps.setString(2, user.getName());
ps.setString(3, user.getEmail());
});
}
}
}
Error Tracking and Custom Metrics
1. Error Tracking
import co.elastic.apm.api.ElasticApm;
import co.elastic.apm.api.Transaction;
@Service
public class ErrorTrackingService {
public void processWithErrorHandling(Order order) {
Transaction transaction = ElasticApm.currentTransaction();
try {
// Business logic
processOrder(order);
} catch (BusinessException e) {
// Capture business exceptions with custom context
transaction.captureException(e);
transaction.addLabel("error.type", "business");
transaction.addLabel("error.code", e.getErrorCode());
transaction.addLabel("order.amount", order.getAmount());
handleBusinessError(e);
} catch (TechnicalException e) {
// Capture technical exceptions
transaction.captureException(e);
transaction.addLabel("error.type", "technical");
transaction.addLabel("error.severity", "high");
handleTechnicalError(e);
throw e;
} catch (Exception e) {
// Capture all other exceptions
transaction.captureException(e);
transaction.addLabel("error.type", "unexpected");
log.error("Unexpected error processing order: {}", order.getId(), e);
throw new RuntimeException("Processing failed", e);
}
}
public void logCustomError(String message, Map<String, String> context) {
Transaction transaction = ElasticApm.currentTransaction();
// Create custom error
Exception customError = new RuntimeException(message);
// Capture with custom context
transaction.captureException(customError);
// Add custom labels
context.forEach(transaction::addLabel);
}
}
2. Custom Metrics
import co.elastic.apm.api.ElasticApm;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.Metrics;
@Service
public class MetricsService {
private final Counter orderCounter;
private final Timer orderProcessingTimer;
private final Counter errorCounter;
public MetricsService() {
this.orderCounter = Counter.builder("orders.processed")
.description("Total number of orders processed")
.register(Metrics.globalRegistry);
this.orderProcessingTimer = Timer.builder("orders.processing.time")
.description("Order processing time")
.register(Metrics.globalRegistry);
this.errorCounter = Counter.builder("orders.errors")
.description("Total number of order processing errors")
.register(Metrics.globalRegistry);
}
public void recordOrderMetrics(Order order, long processingTime) {
Transaction transaction = ElasticApm.currentTransaction();
// Custom labels for the transaction
transaction.addLabel("order.amount", order.getAmount());
transaction.addLabel("order.currency", order.getCurrency());
transaction.addLabel("order.items.count", order.getItems().size());
transaction.addLabel("processing.time.ms", processingTime);
// Micrometer metrics
orderCounter.increment();
orderProcessingTimer.record(processingTime, TimeUnit.MILLISECONDS);
// Business-specific metrics
if (order.getAmount() > 1000) {
transaction.addLabel("order.size", "large");
} else {
transaction.addLabel("order.size", "standard");
}
}
public void recordError(Order order, String errorType) {
errorCounter.increment();
Transaction transaction = ElasticApm.currentTransaction();
transaction.addLabel("error.type", errorType);
transaction.addLabel("order.id", order.getId());
}
}
Advanced Configuration
1. Programmatic Configuration
import co.elastic.apm.attach.ElasticApmAttacher;
import org.springframework.context.annotation.Configuration;
import javax.annotation.PostConstruct;
@Configuration
public class ElasticApmConfig {
@PostConstruct
public void init() {
// Programmatic configuration
Map<String, String> config = new HashMap<>();
config.put("service_name", "order-service");
config.put("application_packages", "com.mycompany.orderservice");
config.put("server_url", "http://localhost:8200");
config.put("environment", "production");
config.put("transaction_sample_rate", "1.0");
config.put("capture_body", "all");
config.put("capture_headers", "true");
config.put("log_level", "INFO");
config.put("ignore_urls", "/health,/metrics,/info");
config.put("disable_instrumentations", "jdbc,redis");
config.put("enable_experimental_instrumentations", "true");
ElasticApmAttacher.attach(config);
}
}
2. Spring Boot Auto-Configuration
# application.yml
elastic:
apm:
service_name: order-service
application_packages: com.mycompany.orderservice
server_url: http://localhost:8200
secret_token: ${APM_SECRET_TOKEN}
environment: ${APM_ENVIRONMENT:development}
# Performance tuning
transaction_sample_rate: 1.0
capture_body: all
capture_headers: true
# Filtering
ignore_urls: /health,/metrics,/info,/actuator/**
disable_instrumentations: jdbc
# Logging
log_level: INFO
log_file: logs/apm.log
# Advanced
enable_experimental_instrumentations: true
metrics_interval: 30s
api_request_time: 10s
Distributed Tracing
1. Microservice Communication
@Service
public class DistributedTracingService {
@Autowired
private RestTemplate restTemplate;
public Order processDistributedOrder(Order order) {
Transaction transaction = ElasticApm.currentTransaction();
try {
// Step 1: Validate order (local)
try (Span validationSpan = transaction.startSpan("validation", "business", "order-validation")) {
validateOrder(order);
}
// Step 2: Process payment (external service)
try (Span paymentSpan = transaction.startSpan("external", "http", "payment-service")) {
PaymentResult payment = callPaymentService(order);
paymentSpan.addLabel("payment.status", payment.getStatus());
}
// Step 3: Update inventory (external service)
try (Span inventorySpan = transaction.startSpan("external", "http", "inventory-service")) {
InventoryUpdateResult inventory = callInventoryService(order);
inventorySpan.addLabel("inventory.updated", inventory.isSuccess());
}
// Step 4: Send notification (async)
try (Span notificationSpan = transaction.startSpan("messaging", "kafka", "notifications")) {
sendOrderNotification(order);
}
return order;
} catch (Exception e) {
transaction.captureException(e);
throw e;
}
}
private PaymentResult callPaymentService(Order order) {
String url = "http://payment-service/api/payments";
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
// Inject distributed tracing headers
ElasticApm.currentTransaction().injectTraceHeaders(headers::set);
HttpEntity<PaymentRequest> request = new HttpEntity<>(
new PaymentRequest(order), headers);
return restTemplate.postForObject(url, request, PaymentResult.class);
}
private InventoryUpdateResult callInventoryService(Order order) {
String url = "http://inventory-service/api/inventory/update";
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
ElasticApm.currentTransaction().injectTraceHeaders(headers::set);
HttpEntity<InventoryUpdateRequest> request = new HttpEntity<>(
new InventoryUpdateRequest(order), headers);
return restTemplate.postForObject(url, request, InventoryUpdateResult.class);
}
private void sendOrderNotification(Order order) {
// Async notification sending
// This will be part of the same trace
}
}
Performance Monitoring
1. Method-Level Monitoring
import co.elastic.apm.api.CaptureSpan;
@Service
public class PerformanceMonitoredService {
@CaptureSpan("databaseQuery")
public List<User> findUsersByCriteria(UserCriteria criteria) {
// This method execution will be captured as a span
return userRepository.findByCriteria(criteria);
}
@CaptureSpan(value = "complexCalculation", type = "business", subtype = "calculation")
public CalculationResult performComplexCalculation(InputData input) {
// Complex business logic
return calculationEngine.calculate(input);
}
// Async method monitoring
@CaptureSpan("asyncProcessing")
public CompletableFuture<ProcessResult> processAsync(ProcessRequest request) {
return CompletableFuture.supplyAsync(() -> {
// This will be traced as part of the same transaction
return heavyProcessing(request);
});
}
}
2. Custom Performance Metrics
@Service
public class PerformanceMetricsService {
private final Map<String, Timer.Sample> activeOperations = new ConcurrentHashMap<>();
public void startOperation(String operationId) {
Timer.Sample sample = Timer.start(Metrics.globalRegistry);
activeOperations.put(operationId, sample);
// Also track in APM
Transaction transaction = ElasticApm.currentTransaction();
transaction.addLabel("operation.id", operationId);
transaction.addLabel("operation.start_time", System.currentTimeMillis());
}
public void endOperation(String operationId, String status) {
Timer.Sample sample = activeOperations.remove(operationId);
if (sample != null) {
sample.stop(Timer.builder("custom.operation.duration")
.tag("operation", operationId)
.tag("status", status)
.register(Metrics.globalRegistry));
}
// Update APM transaction
Transaction transaction = ElasticApm.currentTransaction();
transaction.addLabel("operation.end_time", System.currentTimeMillis());
transaction.addLabel("operation.status", status);
}
public void recordCustomMetric(String name, double value, String... tags) {
// Record custom business metrics
Transaction transaction = ElasticApm.currentTransaction();
transaction.addLabel("metric." + name, value);
// Also record in Micrometer if needed
Metrics.gauge("business." + name,
List.of(Tag.of("environment", "production")),
value);
}
}
Best Practices
1. Naming Conventions
public class ApmNamingConventions {
// Good transaction names
public void goodTransactionNames() {
Transaction transaction = ElasticApm.currentTransaction();
// Use action-oriented names
transaction.setName("ProcessCustomerOrder");
transaction.setName("GenerateMonthlyReport");
transaction.setName("ValidateUserSession");
// Avoid generic names
// transaction.setName("Service"); // Too generic
// transaction.setName("doStuff"); // Not descriptive
}
// Good span names and types
public void goodSpanExamples() {
Transaction transaction = ElasticApm.currentTransaction();
try (Span span = transaction.startSpan("external", "http", "payment-gateway")) {
span.setName("Charge Customer Payment"); // Descriptive name
span.addLabel("payment.amount", 100.0);
span.addLabel("payment.currency", "USD");
}
try (Span span = transaction.startSpan("db", "postgresql", "user-query")) {
span.setName("Find User by Email");
span.addLabel("db.table", "users");
span.addLabel("db.operation", "SELECT");
}
}
}
// Annotation-based naming
@Service
public class WellInstrumentedService {
@CaptureTransaction(type = "Service", value = "OrderProcessing")
public void processOrder(Order order) {
// Clear, business-meaningful name
}
@CaptureSpan("CustomerPaymentValidation")
public void validatePayment(Payment payment) {
// Specific span name
}
}
2. Context and Labels
public class ContextBestPractices {
public void addRelevantContext(Order order, User user) {
Transaction transaction = ElasticApm.currentTransaction();
// Add business-relevant context
transaction.addLabel("order.id", order.getId());
transaction.addLabel("order.total", order.getTotalAmount());
transaction.addLabel("customer.id", user.getId());
transaction.addLabel("customer.tier", user.getTier());
// Add technical context
transaction.addLabel("service.version", "1.2.3");
transaction.addLabel("deployment.environment", "production");
// Avoid sensitive data
// transaction.addLabel("credit_card.number", order.getCreditCardNumber()); // BAD
// transaction.addLabel("user.password", user.getPassword()); // BAD
// Use hashed or masked values for sensitive data
transaction.addLabel("credit_card.last4",
maskCreditCard(order.getCreditCardNumber()));
}
private String maskCreditCard(String cardNumber) {
if (cardNumber == null || cardNumber.length() < 4) {
return "****";
}
return "****" + cardNumber.substring(cardNumber.length() - 4);
}
public void addErrorContext(Exception error, Order order) {
Transaction transaction = ElasticApm.currentTransaction();
transaction.captureException(error);
// Add context that helps debugging
transaction.addLabel("error.type", error.getClass().getSimpleName());
transaction.addLabel("order.status", order.getStatus());
transaction.addLabel("retry.count", order.getRetryCount());
if (error instanceof PaymentException) {
PaymentException pe = (PaymentException) error;
transaction.addLabel("payment.error_code", pe.getErrorCode());
transaction.addLabel("payment.gateway", pe.getGateway());
}
}
}
This comprehensive guide covers Elastic APM Java Agent setup, manual instrumentation, Spring Boot integration, distributed tracing, error tracking, and performance monitoring with best practices for effective application performance management.