MACRO NEPAL
Datadog APM in Java: Comprehensive Application Performance Monitoring
awiskar acharya
November 26, 2025
12:42 pm

Datadog APM provides distributed tracing and performance monitoring for Java applications. It helps you identify bottlenecks, track requests across microservices, and monitor application health.

Setup and Dependencies

1. Maven Dependencies
<properties>
<dd-trace-java.version>1.10.0</dd-trace-java.version>
<spring-boot.version>3.1.0</spring-boot.version>
</properties>
<dependencies>
<!-- Datadog APM Tracer -->
<dependency>
<groupId>com.datadoghq</groupId>
<artifactId>dd-trace-api</artifactId>
<version>${dd-trace-java.version}</version>
</dependency>
<!-- Datadog Java Agent (for auto-instrumentation) -->
<!-- This is typically added as a Java agent, not as a dependency -->
<!-- Spring Boot -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
<version>${spring-boot.version}</version>
</dependency>
</dependencies>
2. Docker Compose for Local Development
version: '3.8'
services:
datadog-agent:
image: gcr.io/datadoghq/agent:latest
environment:
- DD_API_KEY=${DATADOG_API_KEY}
- DD_APM_ENABLED=true
- DD_APM_NON_LOCAL_TRAFFIC=true
- DD_LOGS_ENABLED=true
- DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true
- DD_PROCESS_AGENT_ENABLED=true
ports:
- "8126:8126" # APM
- "8125:8125/udp" # StatsD
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /proc/:/host/proc/:ro
- /sys/fs/cgroup/:/host/sys/fs/cgroup:ro

Configuration

1. Application Properties
# application.yml
spring:
application:
name: user-service
management:
endpoints:
web:
exposure:
include: health,metrics,info,prometheus
metrics:
export:
datadog:
enabled: true
api-key: ${DATADOG_API_KEY}
step: 1m
logging:
level:
com.datadog: DEBUG
2. Java Agent Configuration
# JVM arguments for Datadog agent
java -javaagent:/path/to/dd-java-agent.jar \
-Ddd.service=user-service \
-Ddd.env=production \
-Ddd.version=1.0.0 \
-Ddd.trace.sample.rate=1.0 \
-Ddd.logs.injection=true \
-jar your-application.jar
3. Environment Variables
export DD_SERVICE_NAME=user-service
export DD_ENV=production
export DD_VERSION=1.0.0
export DD_AGENT_HOST=localhost
export DD_TRACE_AGENT_PORT=8126
export DD_LOGS_INJECTION=true
export DD_TRACE_SAMPLE_RATE=1.0
export DD_PROFILING_ENABLED=true
export DD_RUNTIME_METRICS_ENABLED=true

Manual Instrumentation

1. Basic Tracing
import datadog.trace.api.Trace;
import datadog.trace.api.DDTags;
import io.opentracing.Span;
import io.opentracing.Tracer;
import io.opentracing.util.GlobalTracer;
@Service
public class UserService {
private final Tracer tracer = GlobalTracer.get();
private final UserRepository userRepository;
@Trace(operationName = "user.create", resourceName = "UserService.createUser")
public User createUser(CreateUserRequest request) {
Span span = tracer.activeSpan();
try {
// Add custom tags to the span
span.setTag("user.email", request.getEmail());
span.setTag("user.role", request.getRole());
span.setTag(DDTags.SERVICE_NAME, "user-service");
// Business logic
validateUserRequest(request);
User user = userRepository.save(mapToUser(request));
// Log custom event
span.log(Map.of("event", "user_created", "user_id", user.getId()));
return user;
} catch (Exception e) {
// Mark span as error
span.setTag(DDTags.ERROR, true);
span.log(Map.of("error.object", e));
throw e;
}
}
@Trace(operationName = "user.batch_processing", resourceName = "UserService.processBatch")
public void processUserBatch(List<CreateUserRequest> requests) {
Span parentSpan = tracer.activeSpan();
for (int i = 0; i < requests.size(); i++) {
CreateUserRequest request = requests.get(i);
// Create child span for each user
Span childSpan = tracer.buildSpan("user.process_single")
.asChildOf(parentSpan)
.withTag("batch.index", i)
.start();
try (Scope scope = tracer.activateSpan(childSpan)) {
processSingleUser(request);
} catch (Exception e) {
childSpan.setTag(DDTags.ERROR, true);
childSpan.log(Map.of("error.object", e));
} finally {
childSpan.finish();
}
}
}
}
2. Custom Span Creation
@Component
public class DatabaseService {
private final Tracer tracer = GlobalTracer.get();
public User findUserById(String userId) {
Span span = tracer.buildSpan("database.query")
.withTag("db.operation", "SELECT")
.withTag("db.table", "users")
.withTag("db.user_id", userId)
.start();
try (Scope scope = tracer.activateSpan(span)) {
// Simulate database query
Thread.sleep(100);
return userRepository.findById(userId);
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.log(Map.of("error.message", e.getMessage()));
throw new RuntimeException("Database error", e);
} finally {
span.finish();
}
}
public List<User> findUsersByCriteria(UserCriteria criteria) {
return tracer.trace("database.complex_query", () -> {
Span span = tracer.activeSpan();
span.setTag("db.operation", "SELECT");
span.setTag("db.table", "users");
span.setTag("db.criteria", criteria.toString());
// Complex query logic
return executeComplexQuery(criteria);
});
}
}
3. Async Tracing
@Service
public class AsyncUserService {
private final Tracer tracer = GlobalTracer.get();
private final ExecutorService executorService;
@Async
@Trace(operationName = "async.user.processing")
public CompletableFuture<User> processUserAsync(String userId) {
Span span = tracer.activeSpan();
return CompletableFuture.supplyAsync(() -> {
// Activate the span in the async thread
try (Scope scope = tracer.activateSpan(span)) {
span.setTag("user.id", userId);
span.setTag("thread.name", Thread.currentThread().getName());
// Simulate async processing
User user = userRepository.findById(userId);
enrichUserData(user);
return user;
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.log(Map.of("error.object", e));
throw new CompletionException(e);
}
}, executorService);
}
@Trace(operationName = "user.parallel.processing")
public List<User> processUsersInParallel(List<String> userIds) {
Span parentSpan = tracer.activeSpan();
List<CompletableFuture<User>> futures = userIds.stream()
.map(userId -> {
return CompletableFuture.supplyAsync(() -> {
// Create child span for each async operation
Span childSpan = tracer.buildSpan("user.async.process")
.asChildOf(parentSpan)
.withTag("user.id", userId)
.start();
try (Scope scope = tracer.activateSpan(childSpan)) {
return processSingleUser(userId);
} catch (Exception e) {
childSpan.setTag(DDTags.ERROR, true);
throw e;
} finally {
childSpan.finish();
}
}, executorService);
})
.collect(Collectors.toList());
return futures.stream()
.map(CompletableFuture::join)
.collect(Collectors.toList());
}
}
4. HTTP Client Tracing
@Component
public class TracingHttpClient {
private final Tracer tracer = GlobalTracer.get();
private final RestTemplate restTemplate;
@Trace(operationName = "http.client.call")
public <T> T executeWithTracing(String url, HttpMethod method, 
Object request, Class<T> responseType) {
Span span = tracer.activeSpan();
try {
// Add HTTP headers for distributed tracing
HttpHeaders headers = new HttpHeaders();
tracer.inject(span.context(), Format.Builtin.HTTP_HEADERS, new TextMap() {
@Override
public void put(String key, String value) {
headers.add(key, value);
}
@Override
public Iterator<Map.Entry<String, String>> iterator() {
throw new UnsupportedOperationException();
}
});
// Add custom headers
headers.add("X-Correlation-ID", span.context().toTraceId());
HttpEntity<Object> entity = new HttpEntity<>(request, headers);
ResponseEntity<T> response = restTemplate.exchange(url, method, entity, responseType);
// Add response tags to span
span.setTag("http.status_code", response.getStatusCodeValue());
span.setTag("http.url", url);
span.setTag("http.method", method.name());
return response.getBody();
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.setTag("http.error", e.getMessage());
throw e;
}
}
}

Spring Boot Integration

1. Configuration Class
@Configuration
public class DatadogConfig {
@Bean
public Tracer tracer() {
return GlobalTracer.get();
}
@Bean
public Filter datadogFilter() {
return new TracingFilter();
}
@Bean
public RestTemplate tracingRestTemplate(Tracer tracer) {
RestTemplate restTemplate = new RestTemplate();
// Add tracing interceptor
restTemplate.setInterceptors(Collections.singletonList(
new TracingClientHttpRequestInterceptor(tracer)
));
return restTemplate;
}
}
@Component
class TracingClientHttpRequestInterceptor implements ClientHttpRequestInterceptor {
private final Tracer tracer;
public TracingClientHttpRequestInterceptor(Tracer tracer) {
this.tracer = tracer;
}
@Override
public ClientHttpResponse intercept(HttpRequest request, byte[] body, 
ClientHttpRequestExecution execution) throws IOException {
Span span = tracer.buildSpan("http.client")
.withTag("http.method", request.getMethod().name())
.withTag("http.url", request.getURI().toString())
.start();
// Inject tracing headers
tracer.inject(span.context(), Format.Builtin.HTTP_HEADERS, new TextMap() {
@Override
public void put(String key, String value) {
request.getHeaders().add(key, value);
}
@Override
public Iterator<Map.Entry<String, String>> iterator() {
throw new UnsupportedOperationException();
}
});
try (Scope scope = tracer.activateSpan(span)) {
ClientHttpResponse response = execution.execute(request, body);
span.setTag("http.status_code", response.getStatusCode().value());
return response;
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.log(Map.of("error.object", e));
throw e;
} finally {
span.finish();
}
}
}
2. Controller with Custom Tracing
@RestController
@RequestMapping("/api/users")
public class UserController {
private final Tracer tracer = GlobalTracer.get();
private final UserService userService;
private static final Logger logger = LoggerFactory.getLogger(UserController.class);
@PostMapping
@Trace(operationName = "http.request", resourceName = "POST /api/users")
public ResponseEntity<User> createUser(@RequestBody CreateUserRequest request) {
Span span = tracer.activeSpan();
try {
// Add custom tags
span.setTag("http.route", "/api/users");
span.setTag("user.email", request.getEmail());
span.setTag("request.size", request.toString().length());
logger.info("Creating user: {}", request.getEmail());
User user = userService.createUser(request);
span.setTag("user.id", user.getId());
span.setTag("http.status_code", 201);
return ResponseEntity.status(HttpStatus.CREATED).body(user);
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.setTag("http.status_code", 500);
span.log(Map.of("error.message", e.getMessage()));
throw e;
}
}
@GetMapping("/{userId}")
@Trace(operationName = "http.request", resourceName = "GET /api/users/{userId}")
public ResponseEntity<User> getUser(@PathVariable String userId) {
Span span = tracer.activeSpan();
span.setTag("user.id", userId);
try {
User user = userService.findUserById(userId);
if (user == null) {
span.setTag("http.status_code", 404);
return ResponseEntity.notFound().build();
}
span.setTag("http.status_code", 200);
return ResponseEntity.ok(user);
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.setTag("http.status_code", 500);
throw e;
}
}
}
3. AOP for Method Tracing
@Aspect
@Component
public class TracingAspect {
private final Tracer tracer = GlobalTracer.get();
@Around("@annotation(Traced)")
public Object traceMethod(ProceedingJoinPoint joinPoint) throws Throwable {
String methodName = joinPoint.getSignature().toShortString();
Span span = tracer.buildSpan(methodName)
.withTag("class", joinPoint.getTarget().getClass().getSimpleName())
.withTag("method", joinPoint.getSignature().getName())
.start();
try (Scope scope = tracer.activateSpan(span)) {
Object result = joinPoint.proceed();
span.setTag("result.type", result != null ? result.getClass().getSimpleName() : "void");
return result;
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.log(Map.of("error.object", e));
throw e;
} finally {
span.finish();
}
}
}
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface Traced {
String value() default "";
}
4. Database Query Tracing
@Aspect
@Component
public class DatabaseTracingAspect {
private final Tracer tracer = GlobalTracer.get();
@Around("execution(* org.springframework.data.repository.Repository+.*(..))")
public Object traceRepositoryMethods(ProceedingJoinPoint joinPoint) throws Throwable {
String methodName = joinPoint.getSignature().getName();
String repositoryName = joinPoint.getTarget().getClass().getSimpleName();
Span span = tracer.buildSpan("database.repository")
.withTag("db.operation", methodName)
.withTag("db.repository", repositoryName)
.start();
long startTime = System.currentTimeMillis();
try (Scope scope = tracer.activateSpan(span)) {
Object result = joinPoint.proceed();
long duration = System.currentTimeMillis() - startTime;
span.setTag("db.duration_ms", duration);
span.setTag("db.result_count", getResultCount(result));
return result;
} catch (Exception e) {
span.setTag(DDTags.ERROR, true);
span.setTag("db.error", e.getMessage());
throw e;
} finally {
span.finish();
}
}
private int getResultCount(Object result) {
if (result == null) return 0;
if (result instanceof Collection) return ((Collection<?>) result).size();
if (result instanceof Optional) return ((Optional<?>) result).isPresent() ? 1 : 0;
return 1;
}
}

Custom Metrics and Business Monitoring

1. Custom Metrics Service
@Component
public class DatadogMetricsService {
private final StatsDClient statsDClient;
public DatadogMetricsService() {
this.statsDClient = new NonBlockingStatsDClientBuilder()
.prefix("user.service")
.hostname("localhost")
.port(8125)
.build();
}
public void recordUserCreation(String userId, String role, long duration) {
// Increment counter
statsDClient.incrementCounter("user.created", 
"role:" + role, "status:success");
// Record histogram for duration
statsDClient.recordHistogramValue("user.creation.duration", duration,
"role:" + role);
// Record gauge for active users
statsDClient.recordGaugeValue("users.active", getActiveUsersCount());
}
public void recordUserLogin(String userId, boolean success) {
String status = success ? "success" : "failure";
statsDClient.incrementCounter("user.login", 
"user_id:" + userId, "status:" + status);
}
public void recordBusinessEvent(String eventName, Map<String, String> tags) {
statsDClient.incrementCounter("business.event." + eventName,
tags.entrySet().stream()
.map(e -> e.getKey() + ":" + e.getValue())
.toArray(String[]::new));
}
}
2. Business Metrics Integration
@Service
public class OrderService {
private final DatadogMetricsService metricsService;
@Trace(operationName = "order.process")
public Order processOrder(OrderRequest request) {
long startTime = System.currentTimeMillis();
try {
Order order = createOrder(request);
processPayment(order);
updateInventory(order);
long duration = System.currentTimeMillis() - startTime;
// Record business metrics
metricsService.recordHistogramValue("order.processing.duration", duration,
"currency:" + order.getCurrency(),
"payment_method:" + order.getPaymentMethod());
metricsService.incrementCounter("order.completed",
"status:success", "currency:" + order.getCurrency());
return order;
} catch (Exception e) {
metricsService.incrementCounter("order.completed", "status:failure");
throw e;
}
}
}

Error Tracking and Monitoring

1. Custom Error Handler
@ControllerAdvice
public class GlobalExceptionHandler {
private final Tracer tracer = GlobalTracer.get();
@ExceptionHandler(Exception.class)
public ResponseEntity<ErrorResponse> handleException(Exception e, HttpServletRequest request) {
Span span = tracer.activeSpan();
if (span != null) {
span.setTag(DDTags.ERROR, true);
span.log(Map.of(
"error.message", e.getMessage(),
"error.stack", getStackTrace(e),
"http.url", request.getRequestURL().toString(),
"http.method", request.getMethod()
));
}
// Send custom metric for errors
recordErrorMetric(e, request);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(new ErrorResponse("Internal server error"));
}
private void recordErrorMetric(Exception e, HttpServletRequest request) {
// This would integrate with Datadog metrics
System.err.printf("Error occurred: %s at %s %s%n", 
e.getMessage(), request.getMethod(), request.getRequestURI());
}
private String getStackTrace(Exception e) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
return sw.toString();
}
}
2. Health Check with Metrics
@Component
public class DatabaseHealthIndicator implements HealthIndicator {
private final DatadogMetricsService metricsService;
private final UserRepository userRepository;
@Override
public Health health() {
long startTime = System.currentTimeMillis();
try {
// Simple database check
userRepository.count();
long duration = System.currentTimeMillis() - startTime;
metricsService.recordGaugeValue("health.database.duration", duration);
metricsService.incrementCounter("health.check", "component:database", "status:healthy");
return Health.up()
.withDetail("database", "connected")
.withDetail("response_time", duration + "ms")
.build();
} catch (Exception e) {
long duration = System.currentTimeMillis() - startTime;
metricsService.incrementCounter("health.check", "component:database", "status:unhealthy");
metricsService.recordGaugeValue("health.database.duration", duration);
return Health.down(e)
.withDetail("database", "disconnected")
.withDetail("response_time", duration + "ms")
.build();
}
}
}

Testing and Verification

1. Test Configuration
@SpringBootTest
@TestPropertySource(properties = {
"dd.trace.enabled=false",
"dd.env=test",
"dd.service.name=user-service-test"
})
public class UserServiceTest {
@Test
public void testUserCreationWithTracing() {
// Test without actual tracing in test environment
UserService userService = new UserService();
User user = userService.createUser(createTestRequest());
assertNotNull(user);
assertNotNull(user.getId());
}
}
2. Tracing Verification Utility
@Component
@Profile("dev")
public class TracingVerifier {
private final Tracer tracer = GlobalTracer.get();
public void verifyTrace(String operationName) {
Span span = tracer.activeSpan();
if (span != null) {
System.out.printf("Active span: %s, Trace ID: %s%n", 
span.getOperationName(), span.context().toTraceId());
} else {
System.out.println("No active span");
}
}
}

Best Practices

  1. Consistent Naming: Use consistent operation names and resource names
  2. Reasonable Sampling: Adjust sampling rate based on traffic
  3. Tag Wisely: Avoid high-cardinality tags that can impact performance
  4. Error Handling: Always mark spans as errors when exceptions occur
  5. Resource Cleanup: Ensure spans are properly finished
  6. Async Context: Properly propagate context to async operations
  7. Security: Avoid storing sensitive data in tags
// Example of secure tagging
public void setSecureTags(Span span, User user) {
span.setTag("user.id", user.getId());
span.setTag("user.role", user.getRole());
// Don't include sensitive information
// span.setTag("user.email", user.getEmail()); // Avoid
// span.setTag("user.password_hash", user.getPasswordHash()); // Avoid
}

Conclusion

Datadog APM in Java provides:

  • Distributed tracing across microservices
  • Performance monitoring and bottleneck identification
  • Custom metrics for business monitoring
  • Error tracking and alerting
  • Integration with existing Spring Boot applications

By implementing the patterns shown above, you can gain deep visibility into your Java applications, identify performance issues, and monitor your system's health in production environments. The combination of automatic instrumentation and custom tracing gives you complete control over what you monitor and how you track your application's performance.

Category : JAVA

Leave a Reply

Your email address will not be published. Required fields are marked *


Macro Nepal Helper