MACRO NEPAL
Buildkite Agent in Java: Complete Guide
awiskar acharya
November 19, 2025
10:41 am

Introduction

Buildkite is a popular CI/CD platform that uses agents to run build jobs. This guide covers how to implement a Buildkite agent in Java, including job execution, artifact handling, and pipeline integration.

Dependencies and Setup

1. Maven Dependencies

<properties>
<spring-boot.version>3.2.0</spring-boot.version>
<okhttp.version>4.12.0</okhttp.version>
<jackson.version>2.15.3</jackson.version>
<jsch.version>0.2.15</jsch.version>
</properties>
<dependencies>
<!-- Spring Boot -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<!-- HTTP Client -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
<!-- JSON Processing -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<!-- SSH for Git operations -->
<dependency>
<groupId>com.github.mwiede</groupId>
<artifactId>jsch</artifactId>
<version>${jsch.version}</version>
</dependency>
<!-- Process Execution -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.4.0</version>
</dependency>
<!-- File System Watcher -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.15.1</version>
</dependency>
<!-- Testing -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${spring-boot.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

Core Models

2. Buildkite API Models

package com.example.buildkite.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
public class BuildkiteJob {
private String id;
private String state;
private String type;
private Map<String, Object> env;
private List<String> command;
private String artifactPaths;
private String priority;
private String buildId;
private String buildUrl;
private String webUrl;
private String logUrl;
private String rawLogUrl;
private Map<String, String> step;
private Agent agent;
private OffsetDateTime createdAt;
private OffsetDateTime scheduledAt;
private OffsetDateTime startedAt;
private OffsetDateTime finishedAt;
// Getters and Setters
public String getId() { return id; }
public void setId(String id) { this.id = id; }
public String getState() { return state; }
public void setState(String state) { this.state = state; }
public String getType() { return type; }
public void setType(String type) { this.type = type; }
public Map<String, Object> getEnv() { return env; }
public void setEnv(Map<String, Object> env) { this.env = env; }
public List<String> getCommand() { return command; }
public void setCommand(List<String> command) { this.command = command; }
public String getArtifactPaths() { return artifactPaths; }
public void setArtifactPaths(String artifactPaths) { this.artifactPaths = artifactPaths; }
public String getPriority() { return priority; }
public void setPriority(String priority) { this.priority = priority; }
public String getBuildId() { return buildId; }
public void setBuildId(String buildId) { this.buildId = buildId; }
public String getBuildUrl() { return buildUrl; }
public void setBuildUrl(String buildUrl) { this.buildUrl = buildUrl; }
public String getWebUrl() { return webUrl; }
public void setWebUrl(String webUrl) { this.webUrl = webUrl; }
public String getLogUrl() { return logUrl; }
public void setLogUrl(String logUrl) { this.logUrl = logUrl; }
public String getRawLogUrl() { return rawLogUrl; }
public void setRawLogUrl(String rawLogUrl) { this.rawLogUrl = rawLogUrl; }
public Map<String, String> getStep() { return step; }
public void setStep(Map<String, String> step) { this.step = step; }
public Agent getAgent() { return agent; }
public void setAgent(Agent agent) { this.agent = agent; }
public OffsetDateTime getCreatedAt() { return createdAt; }
public void setCreatedAt(OffsetDateTime createdAt) { this.createdAt = createdAt; }
public OffsetDateTime getScheduledAt() { return scheduledAt; }
public void setScheduledAt(OffsetDateTime scheduledAt) { this.scheduledAt = scheduledAt; }
public OffsetDateTime getStartedAt() { return startedAt; }
public void setStartedAt(OffsetDateTime startedAt) { this.startedAt = startedAt; }
public OffsetDateTime getFinishedAt() { return finishedAt; }
public void setFinishedAt(OffsetDateTime finishedAt) { this.finishedAt = finishedAt; }
public static class Agent {
private String id;
private String name;
private String version;
private String hostname;
private String pid;
// Getters and Setters
public String getId() { return id; }
public void setId(String id) { this.id = id; }
public String getName() { return name; }
public void setName(String name) { this.name = name; }
public String getVersion() { return version; }
public void setVersion(String version) { this.version = version; }
public String getHostname() { return hostname; }
public void setHostname(String hostname) { this.hostname = hostname; }
public String getPid() { return pid; }
public void setPid(String pid) { this.pid = pid; }
}
}
package com.example.buildkite.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
public class BuildkiteAgent {
private String id;
private String name;
private String version;
private String hostname;
private String ipAddress;
private String userAgent;
private String pid;
private String state;
private String buildPath;
private String tags;
private List<String> metaData;
private String job;
private String lastJobFinishedAt;
private String priority;
private String createdAt;
private String disconnectedAt;
private String connectedAt;
private String heartbeatAt;
// Getters and Setters
public String getId() { return id; }
public void setId(String id) { this.id = id; }
public String getName() { return name; }
public void setName(String name) { this.name = name; }
public String getVersion() { return version; }
public void setVersion(String version) { this.version = version; }
public String getHostname() { return hostname; }
public void setHostname(String hostname) { this.hostname = hostname; }
public String getIpAddress() { return ipAddress; }
public void setIpAddress(String ipAddress) { this.ipAddress = ipAddress; }
public String getUserAgent() { return userAgent; }
public void setUserAgent(String userAgent) { this.userAgent = userAgent; }
public String getPid() { return pid; }
public void setPid(String pid) { this.pid = pid; }
public String getState() { return state; }
public void setState(String state) { this.state = state; }
public String getBuildPath() { return buildPath; }
public void setBuildPath(String buildPath) { this.buildPath = buildPath; }
public String getTags() { return tags; }
public void setTags(String tags) { this.tags = tags; }
public List<String> getMetaData() { return metaData; }
public void setMetaData(List<String> metaData) { this.metaData = metaData; }
public String getJob() { return job; }
public void setJob(String job) { this.job = job; }
public String getLastJobFinishedAt() { return lastJobFinishedAt; }
public void setLastJobFinishedAt(String lastJobFinishedAt) { this.lastJobFinishedAt = lastJobFinishedAt; }
public String getPriority() { return priority; }
public void setPriority(String priority) { this.priority = priority; }
public String getCreatedAt() { return createdAt; }
public void setCreatedAt(String createdAt) { this.createdAt = createdAt; }
public String getDisconnectedAt() { return disconnectedAt; }
public void setDisconnectedAt(String disconnectedAt) { this.disconnectedAt = disconnectedAt; }
public String getConnectedAt() { return connectedAt; }
public void setConnectedAt(String connectedAt) { this.connectedAt = connectedAt; }
public String getHeartbeatAt() { return heartbeatAt; }
public void setHeartbeatAt(String heartbeatAt) { this.heartbeatAt = heartbeatAt; }
}
package com.example.buildkite.model;
import java.util.List;
import java.util.Map;
public class BuildkiteArtifact {
private String id;
private String jobId;
private String url;
private String downloadUrl;
private String state;
private String path;
private String absolutePath;
private String globPath;
private long fileSize;
private String sha1sum;
private String mimeType;
private String originalPath;
private Map<String, String> metaData;
// Getters and Setters
public String getId() { return id; }
public void setId(String id) { this.id = id; }
public String getJobId() { return jobId; }
public void setJobId(String jobId) { this.jobId = jobId; }
public String getUrl() { return url; }
public void setUrl(String url) { this.url = url; }
public String getDownloadUrl() { return downloadUrl; }
public void setDownloadUrl(String downloadUrl) { this.downloadUrl = downloadUrl; }
public String getState() { return state; }
public void setState(String state) { this.state = state; }
public String getPath() { return path; }
public void setPath(String path) { this.path = path; }
public String getAbsolutePath() { return absolutePath; }
public void setAbsolutePath(String absolutePath) { this.absolutePath = absolutePath; }
public String getGlobPath() { return globPath; }
public void setGlobPath(String globPath) { this.globPath = globPath; }
public long getFileSize() { return fileSize; }
public void setFileSize(long fileSize) { this.fileSize = fileSize; }
public String getSha1sum() { return sha1sum; }
public void setSha1sum(String sha1sum) { this.sha1sum = sha1sum; }
public String getMimeType() { return mimeType; }
public void setMimeType(String mimeType) { this.mimeType = mimeType; }
public String getOriginalPath() { return originalPath; }
public void setOriginalPath(String originalPath) { this.originalPath = originalPath; }
public Map<String, String> getMetaData() { return metaData; }
public void setMetaData(Map<String, String> metaData) { this.metaData = metaData; }
}

Core Services

3. Buildkite API Client

package com.example.buildkite.client;
import com.example.buildkite.model.BuildkiteAgent;
import com.example.buildkite.model.BuildkiteJob;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Component
public class BuildkiteApiClient {
private static final Logger log = LoggerFactory.getLogger(BuildkiteApiClient.class);
private static final String BASE_URL = "https://api.buildkite.com/v2";
private static final MediaType JSON_MEDIA_TYPE = MediaType.parse("application/json");
private final OkHttpClient httpClient;
private final ObjectMapper objectMapper;
private final String apiToken;
private final String organizationSlug;
public BuildkiteApiClient(
@Value("${buildkite.api-token}") String apiToken,
@Value("${buildkite.organization}") String organizationSlug,
ObjectMapper objectMapper) {
this.apiToken = apiToken;
this.organizationSlug = organizationSlug;
this.objectMapper = objectMapper;
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(300, TimeUnit.SECONDS) // Longer timeout for job execution
.writeTimeout(30, TimeUnit.SECONDS)
.build();
}
public List<BuildkiteJob> getPendingJobs(String agentId) throws IOException {
String url = String.format("%s/organizations/%s/agents/%s/jobs", 
BASE_URL, organizationSlug, agentId);
Request request = new Request.Builder()
.url(url)
.get()
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (response.isSuccessful() && response.body() != null) {
String responseBody = response.body().string();
return objectMapper.readValue(responseBody, 
objectMapper.getTypeFactory().constructCollectionType(List.class, BuildkiteJob.class));
} else {
throw new BuildkiteException("Failed to get pending jobs: " + response.code());
}
}
}
public BuildkiteJob acceptJob(String jobId, String agentId) throws IOException {
String url = String.format("%s/organizations/%s/agents/%s/jobs/%s/accept", 
BASE_URL, organizationSlug, agentId, jobId);
Request request = new Request.Builder()
.url(url)
.put(RequestBody.create("", JSON_MEDIA_TYPE))
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (response.isSuccessful() && response.body() != null) {
String responseBody = response.body().string();
return objectMapper.readValue(responseBody, BuildkiteJob.class);
} else {
throw new BuildkiteException("Failed to accept job: " + response.code());
}
}
}
public void startJob(String jobId, String agentId) throws IOException {
String url = String.format("%s/organizations/%s/agents/%s/jobs/%s/start", 
BASE_URL, organizationSlug, agentId, jobId);
Request request = new Request.Builder()
.url(url)
.put(RequestBody.create("", JSON_MEDIA_TYPE))
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new BuildkiteException("Failed to start job: " + response.code());
}
}
}
public void finishJob(String jobId, String agentId, int exitCode, String output) throws IOException {
String url = String.format("%s/organizations/%s/agents/%s/jobs/%s/finish", 
BASE_URL, organizationSlug, agentId, jobId);
FinishJobRequest finishRequest = new FinishJobRequest(exitCode, output);
String jsonBody = objectMapper.writeValueAsString(finishRequest);
Request request = new Request.Builder()
.url(url)
.put(RequestBody.create(jsonBody, JSON_MEDIA_TYPE))
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new BuildkiteException("Failed to finish job: " + response.code());
}
}
}
public void uploadJobLog(String jobId, String agentId, String logContent) throws IOException {
String url = String.format("%s/organizations/%s/agents/%s/jobs/%s/log", 
BASE_URL, organizationSlug, agentId, jobId);
UploadLogRequest logRequest = new UploadLogRequest(logContent);
String jsonBody = objectMapper.writeValueAsString(logRequest);
Request request = new Request.Builder()
.url(url)
.post(RequestBody.create(jsonBody, JSON_MEDIA_TYPE))
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.warn("Failed to upload job log: {}", response.code());
}
}
}
public BuildkiteAgent registerAgent(AgentRegistrationRequest registration) throws IOException {
String url = String.format("%s/organizations/%s/agents", BASE_URL, organizationSlug);
String jsonBody = objectMapper.writeValueAsString(registration);
Request request = new Request.Builder()
.url(url)
.post(RequestBody.create(jsonBody, JSON_MEDIA_TYPE))
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (response.isSuccessful() && response.body() != null) {
String responseBody = response.body().string();
return objectMapper.readValue(responseBody, BuildkiteAgent.class);
} else {
throw new BuildkiteException("Failed to register agent: " + response.code());
}
}
}
public void sendHeartbeat(String agentId) throws IOException {
String url = String.format("%s/organizations/%s/agents/%s/heartbeat", 
BASE_URL, organizationSlug, agentId);
Request request = new Request.Builder()
.url(url)
.post(RequestBody.create("", JSON_MEDIA_TYPE))
.addHeader("Authorization", "Bearer " + apiToken)
.addHeader("Content-Type", "application/json")
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.warn("Failed to send heartbeat: {}", response.code());
}
}
}
// Request DTOs
public static class FinishJobRequest {
private final int exitStatus;
private final String output;
public FinishJobRequest(int exitStatus, String output) {
this.exitStatus = exitStatus;
this.output = output;
}
public int getExitStatus() { return exitStatus; }
public String getOutput() { return output; }
}
public static class UploadLogRequest {
private final String output;
public UploadLogRequest(String output) {
this.output = output;
}
public String getOutput() { return output; }
}
public static class AgentRegistrationRequest {
private String name;
private String version;
private String hostname;
private String pid;
private String[] tags;
private String[] metaData;
private String buildPath;
private int priority;
// Getters and Setters
public String getName() { return name; }
public void setName(String name) { this.name = name; }
public String getVersion() { return version; }
public void setVersion(String version) { this.version = version; }
public String getHostname() { return hostname; }
public void setHostname(String hostname) { this.hostname = hostname; }
public String getPid() { return pid; }
public void setPid(String pid) { this.pid = pid; }
public String[] getTags() { return tags; }
public void setTags(String[] tags) { this.tags = tags; }
public String[] getMetaData() { return metaData; }
public void setMetaData(String[] metaData) { this.metaData = metaData; }
public String getBuildPath() { return buildPath; }
public void setBuildPath(String buildPath) { this.buildPath = buildPath; }
public int getPriority() { return priority; }
public void setPriority(int priority) { this.priority = priority; }
}
public static class BuildkiteException extends RuntimeException {
public BuildkiteException(String message) {
super(message);
}
public BuildkiteException(String message, Throwable cause) {
super(message, cause);
}
}
}

4. Job Execution Service

package com.example.buildkite.service;
import com.example.buildkite.model.BuildkiteJob;
import org.apache.commons.exec.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@Service
public class JobExecutionService {
private static final Logger log = LoggerFactory.getLogger(JobExecutionService.class);
public JobExecutionResult executeJob(BuildkiteJob job, String workspace) throws IOException {
log.info("Executing job {} in workspace: {}", job.getId(), workspace);
// Create workspace directory
Path workspacePath = Paths.get(workspace, "build-" + job.getBuildId(), job.getId());
Files.createDirectories(workspacePath);
// Set up environment
Map<String, String> environment = createEnvironment(job, workspacePath.toString());
// Execute commands
StringBuilder output = new StringBuilder();
int exitCode = 0;
for (String command : job.getCommand()) {
log.info("Executing command: {}", command);
CommandExecutionResult result = executeCommand(command, workspacePath.toFile(), environment);
output.append(result.getOutput()).append("\n");
if (result.getExitCode() != 0) {
exitCode = result.getExitCode();
log.error("Command failed with exit code: {}", exitCode);
break;
}
}
// Handle artifacts
if (job.getArtifactPaths() != null && exitCode == 0) {
uploadArtifacts(job, workspacePath);
}
return new JobExecutionResult(exitCode, output.toString());
}
private CommandExecutionResult executeCommand(String command, File workingDir, 
Map<String, String> environment) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
CommandLine commandLine = CommandLine.parse(command);
DefaultExecutor executor = new DefaultExecutor();
executor.setWorkingDirectory(workingDir);
executor.setStreamHandler(streamHandler);
executor.setWatchdog(new ExecuteWatchdog(TimeUnit.HOURS.toMillis(2))); // 2-hour timeout
try {
int exitCode = executor.execute(commandLine, environment);
String output = outputStream.toString();
return new CommandExecutionResult(exitCode, output);
} catch (ExecuteException e) {
String output = outputStream.toString();
return new CommandExecutionResult(e.getExitValue(), output);
}
}
private Map<String, String> createEnvironment(BuildkiteJob job, String workspace) {
Map<String, String> env = System.getenv();
// Add Buildkite-specific environment variables
env.put("BUILDKITE", "true");
env.put("BUILDKITE_AGENT_NAME", job.getAgent().getName());
env.put("BUILDKITE_BUILD_ID", job.getBuildId());
env.put("BUILDKITE_BUILD_URL", job.getBuildUrl());
env.put("BUILDKITE_JOB_ID", job.getId());
env.put("BUILDKITE_COMMAND", String.join(" ", job.getCommand()));
env.put("BUILDKITE_ARTIFACT_PATHS", job.getArtifactPaths() != null ? job.getArtifactPaths() : "");
env.put("BUILDKITE_BUILD_PATH", workspace);
// Add job-specific environment variables
if (job.getEnv() != null) {
job.getEnv().forEach((key, value) -> {
if (value != null) {
env.put(key, value.toString());
}
});
}
return env;
}
private void uploadArtifacts(BuildkiteJob job, Path workspace) {
// Implement artifact upload logic
// This would use the Buildkite API to upload artifacts
log.info("Uploading artifacts for job {} from paths: {}", job.getId(), job.getArtifactPaths());
// For now, just log the artifact paths
// In a real implementation, you would:
// 1. Glob match the artifact paths
// 2. Upload each artifact to Buildkite
// 3. Update the job with artifact metadata
}
public static class JobExecutionResult {
private final int exitCode;
private final String output;
public JobExecutionResult(int exitCode, String output) {
this.exitCode = exitCode;
this.output = output;
}
public int getExitCode() { return exitCode; }
public String getOutput() { return output; }
}
public static class CommandExecutionResult {
private final int exitCode;
private final String output;
public CommandExecutionResult(int exitCode, String output) {
this.exitCode = exitCode;
this.output = output;
}
public int getExitCode() { return exitCode; }
public String getOutput() { return output; }
}
}

5. Buildkite Agent Service

package com.example.buildkite.service;
import com.example.buildkite.client.BuildkiteApiClient;
import com.example.buildkite.model.BuildkiteAgent;
import com.example.buildkite.model.BuildkiteJob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
@Service
public class BuildkiteAgentService {
private static final Logger log = LoggerFactory.getLogger(BuildkiteAgentService.class);
private final BuildkiteApiClient apiClient;
private final JobExecutionService jobExecutionService;
private final ExecutorService executorService;
private BuildkiteAgent agent;
private final AtomicBoolean running = new AtomicBoolean(false);
private final AtomicBoolean processingJob = new AtomicBoolean(false);
@Value("${buildkite.agent.name:java-agent}")
private String agentName;
@Value("${buildkite.agent.tags:java,linux}")
private String agentTags;
@Value("${buildkite.workspace:/tmp/buildkite}")
private String workspace;
public BuildkiteAgentService(BuildkiteApiClient apiClient, JobExecutionService jobExecutionService) {
this.apiClient = apiClient;
this.jobExecutionService = jobExecutionService;
this.executorService = Executors.newFixedThreadPool(3); // Allow 3 concurrent jobs
}
public void start() {
if (running.compareAndSet(false, true)) {
log.info("Starting Buildkite agent: {}", agentName);
registerAgent();
startHeartbeat();
startJobPolling();
}
}
public void stop() {
if (running.compareAndSet(true, false)) {
log.info("Stopping Buildkite agent: {}", agentName);
executorService.shutdown();
}
}
private void registerAgent() {
try {
BuildkiteApiClient.AgentRegistrationRequest registration = 
new BuildkiteApiClient.AgentRegistrationRequest();
registration.setName(agentName);
registration.setVersion("1.0.0");
registration.setHostname(getHostname());
registration.setPid(String.valueOf(ProcessHandle.current().pid()));
registration.setTags(agentTags.split(","));
registration.setMetaData(new String[]{"java", "custom-agent"});
registration.setBuildPath(workspace);
registration.setPriority(1);
agent = apiClient.registerAgent(registration);
log.info("Registered agent with ID: {}", agent.getId());
} catch (IOException e) {
log.error("Failed to register agent", e);
throw new RuntimeException("Agent registration failed", e);
}
}
@Scheduled(fixedRate = 30000) // Every 30 seconds
private void startHeartbeat() {
if (running.get() && agent != null) {
try {
apiClient.sendHeartbeat(agent.getId());
log.debug("Sent heartbeat for agent: {}", agent.getId());
} catch (IOException e) {
log.warn("Failed to send heartbeat", e);
}
}
}
private void startJobPolling() {
new Thread(() -> {
while (running.get()) {
try {
pollForJobs();
Thread.sleep(5000); // Poll every 5 seconds
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
} catch (Exception e) {
log.error("Error in job polling", e);
}
}
}, "job-polling-thread").start();
}
private void pollForJobs() {
if (processingJob.get()) {
return; // Skip polling if we're already processing a job
}
try {
List<BuildkiteJob> pendingJobs = apiClient.getPendingJobs(agent.getId());
for (BuildkiteJob job : pendingJobs) {
if (running.get() && !processingJob.get()) {
processJob(job);
}
}
} catch (IOException e) {
log.error("Failed to get pending jobs", e);
}
}
private void processJob(BuildkiteJob job) {
if (!processingJob.compareAndSet(false, true)) {
return; // Another thread is already processing a job
}
executorService.submit(() -> {
try {
executeJob(job);
} finally {
processingJob.set(false);
}
});
}
private void executeJob(BuildkiteJob job) {
log.info("Processing job: {}", job.getId());
try {
// Accept the job
BuildkiteJob acceptedJob = apiClient.acceptJob(job.getId(), agent.getId());
log.info("Accepted job: {}", acceptedJob.getId());
// Start the job
apiClient.startJob(acceptedJob.getId(), agent.getId());
log.info("Started job: {}", acceptedJob.getId());
// Execute the job
JobExecutionService.JobExecutionResult result = 
jobExecutionService.executeJob(acceptedJob, workspace);
// Upload log output
apiClient.uploadJobLog(acceptedJob.getId(), agent.getId(), result.getOutput());
// Finish the job
apiClient.finishJob(acceptedJob.getId(), agent.getId(), result.getExitCode(), result.getOutput());
log.info("Finished job: {} with exit code: {}", acceptedJob.getId(), result.getExitCode());
} catch (Exception e) {
log.error("Failed to process job: {}", job.getId(), e);
try {
// Try to finish the job with error
apiClient.finishJob(job.getId(), agent.getId(), 1, 
"Job execution failed: " + e.getMessage());
} catch (IOException finishError) {
log.error("Failed to finish failed job: {}", job.getId(), finishError);
}
}
}
private String getHostname() {
try {
return java.net.InetAddress.getLocalHost().getHostName();
} catch (Exception e) {
return "unknown-host";
}
}
public BuildkiteAgent getAgent() {
return agent;
}
public boolean isRunning() {
return running.get();
}
public boolean isProcessingJob() {
return processingJob.get();
}
}

6. Git Service for Repository Operations

package com.example.buildkite.service;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.io.File;
import java.nio.file.Path;
@Service
public class GitService {
private static final Logger log = LoggerFactory.getLogger(GitService.class);
public void cloneRepository(String repositoryUrl, String branch, String destination, 
String username, String password) throws GitAPIException {
log.info("Cloning repository: {} branch: {} to: {}", repositoryUrl, branch, destination);
File destinationDir = new File(destination);
Git.cloneRepository()
.setURI(repositoryUrl)
.setBranch(branch)
.setDirectory(destinationDir)
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password))
.call()
.close();
log.info("Successfully cloned repository to: {}", destination);
}
public void checkoutBranch(String repositoryPath, String branch) throws GitAPIException {
log.info("Checking out branch: {} in repository: {}", branch, repositoryPath);
try (Git git = Git.open(new File(repositoryPath))) {
git.checkout()
.setName(branch)
.call();
} catch (Exception e) {
throw new GitAPIException("Failed to checkout branch", e);
}
}
public void pullLatest(String repositoryPath, String username, String password) throws GitAPIException {
log.info("Pulling latest changes for repository: {}", repositoryPath);
try (Git git = Git.open(new File(repositoryPath))) {
git.pull()
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password))
.call();
} catch (Exception e) {
throw new GitAPIException("Failed to pull latest changes", e);
}
}
public String getCurrentCommitHash(String repositoryPath) throws GitAPIException {
try (Git git = Git.open(new File(repositoryPath))) {
return git.getRepository().resolve("HEAD").getName();
} catch (Exception e) {
throw new GitAPIException("Failed to get current commit hash", e);
}
}
public static class GitAPIException extends Exception {
public GitAPIException(String message) {
super(message);
}
public GitAPIException(String message, Throwable cause) {
super(message, cause);
}
}
}

Configuration

7. Application Configuration

package com.example.buildkite.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
@Configuration
@EnableScheduling
public class AppConfig {
@Bean
public com.fasterxml.jackson.databind.ObjectMapper objectMapper() {
return new com.fasterxml.jackson.databind.ObjectMapper();
}
}

8. Application Properties

# application.yml
buildkite:
api-token: ${BUILDKITE_API_TOKEN:your-api-token-here}
organization: ${BUILDKITE_ORGANIZATION:your-organization}
agent:
name: ${BUILDKITE_AGENT_NAME:java-buildkite-agent}
tags: ${BUILDKITE_AGENT_TAGS:java,linux,docker}
workspace: ${BUILDKITE_WORKSPACE:/tmp/buildkite-workspace}
server:
port: 8080
management:
endpoints:
web:
exposure:
include: health,metrics,info
endpoint:
health:
show-details: always
logging:
level:
com.example.buildkite: INFO
org.apache.commons.exec: WARN

REST API Controllers

9. Agent Management API

package com.example.buildkite.controller;
import com.example.buildkite.model.BuildkiteAgent;
import com.example.buildkite.service.BuildkiteAgentService;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.Map;
@RestController
@RequestMapping("/api/agent")
public class AgentController {
private final BuildkiteAgentService agentService;
public AgentController(BuildkiteAgentService agentService) {
this.agentService = agentService;
}
@PostMapping("/start")
public ResponseEntity<Map<String, String>> startAgent() {
agentService.start();
return ResponseEntity.ok(Map.of(
"status", "started",
"agentName", agentService.getAgent() != null ? agentService.getAgent().getName() : "unknown"
));
}
@PostMapping("/stop")
public ResponseEntity<Map<String, String>> stopAgent() {
agentService.stop();
return ResponseEntity.ok(Map.of(
"status", "stopped"
));
}
@GetMapping("/status")
public ResponseEntity<Map<String, Object>> getAgentStatus() {
BuildkiteAgent agent = agentService.getAgent();
return ResponseEntity.ok(Map.of(
"running", agentService.isRunning(),
"processingJob", agentService.isProcessingJob(),
"agent", agent != null ? Map.of(
"id", agent.getId(),
"name", agent.getName(),
"hostname", agent.getHostname(),
"version", agent.getVersion()
) : "not-registered"
));
}
@GetMapping("/info")
public ResponseEntity<BuildkiteAgent> getAgentInfo() {
BuildkiteAgent agent = agentService.getAgent();
if (agent != null) {
return ResponseEntity.ok(agent);
} else {
return ResponseEntity.notFound().build();
}
}
}

Main Application

10. Spring Boot Application

package com.example.buildkite;
import com.example.buildkite.service.BuildkiteAgentService;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class BuildkiteAgentApplication implements CommandLineRunner {
private final BuildkiteAgentService agentService;
public BuildkiteAgentApplication(BuildkiteAgentService agentService) {
this.agentService = agentService;
}
public static void main(String[] args) {
SpringApplication.run(BuildkiteAgentApplication.class, args);
}
@Override
public void run(String... args) throws Exception {
// Auto-start the agent if configured
String autoStart = System.getenv("BUILDKITE_AUTO_START");
if ("true".equalsIgnoreCase(autoStart)) {
agentService.start();
}
}
}

Health Checks

11. Agent Health Indicator

package com.example.buildkite.health;
import com.example.buildkite.service.BuildkiteAgentService;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;
@Component
public class BuildkiteAgentHealthIndicator implements HealthIndicator {
private final BuildkiteAgentService agentService;
public BuildkiteAgentHealthIndicator(BuildkiteAgentService agentService) {
this.agentService = agentService;
}
@Override
public Health health() {
try {
if (!agentService.isRunning()) {
return Health.down()
.withDetail("status", "agent-stopped")
.build();
}
if (agentService.getAgent() == null) {
return Health.down()
.withDetail("status", "agent-not-registered")
.build();
}
return Health.up()
.withDetail("status", "running")
.withDetail("agentId", agentService.getAgent().getId())
.withDetail("processingJob", agentService.isProcessingJob())
.build();
} catch (Exception e) {
return Health.down(e)
.withDetail("status", "error")
.build();
}
}
}

Testing

12. Unit Tests

package com.example.buildkite.test;
import com.example.buildkite.client.BuildkiteApiClient;
import com.example.buildkite.model.BuildkiteJob;
import com.example.buildkite.service.BuildkiteAgentService;
import com.example.buildkite.service.JobExecutionService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Collections;
import java.util.List;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
@ExtendWith(MockitoExtension.class)
class BuildkiteAgentServiceTest {
@Mock
private BuildkiteApiClient apiClient;
@Mock
private JobExecutionService jobExecutionService;
@Test
void testAgentStartAndJobProcessing() throws Exception {
// Given
BuildkiteAgentService agentService = new BuildkiteAgentService(apiClient, jobExecutionService);
BuildkiteJob testJob = new BuildkiteJob();
testJob.setId("test-job-123");
testJob.setCommand(List.of("echo", "hello world"));
when(apiClient.getPendingJobs(anyString())).thenReturn(List.of(testJob));
when(apiClient.acceptJob(anyString(), anyString())).thenReturn(testJob);
when(jobExecutionService.executeJob(any(), anyString()))
.thenReturn(new JobExecutionService.JobExecutionResult(0, "hello world\n"));
// When
agentService.start();
Thread.sleep(1000); // Give time for polling to start
// Then
verify(apiClient, atLeastOnce()).getPendingJobs(anyString());
// Note: In a real test, you'd use proper synchronization
}
}

Docker Support

13. Dockerfile

FROM eclipse-temurin:17-jre
WORKDIR /app
# Install dependencies
RUN apt-get update && apt-get install -y \
git \
ssh \
&& rm -rf /var/lib/apt/lists/*
# Copy application JAR
COPY target/buildkite-agent-*.jar app.jar
# Create workspace directory
RUN mkdir -p /workspace
# Environment variables
ENV BUILDKITE_WORKSPACE=/workspace
ENV BUILDKITE_AUTO_START=true
# Expose health check port
EXPOSE 8080
# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8080/actuator/health || exit 1
# Start application
ENTRYPOINT ["java", "-jar", "app.jar"]

Usage Examples

14. Example Pipeline Configuration

# .buildkite/pipeline.yml
steps:
- label: ":java: Build and Test"
command:
- "./gradlew build"
- "./gradlew test"
agents:
- "java=true"
artifact_paths:
- "build/libs/*.jar"
- "build/reports/**/*"
- label: ":docker: Build Docker Image"
command: 
- "docker build -t myapp:${BUILDKITE_COMMIT} ."
agents:
- "docker=true"
depends_on: ":java: Build and Test"
- label: ":rocket: Deploy to Staging"
command: "./scripts/deploy-staging.sh"
agents:
- "deployment=true"
depends_on: ":docker: Build Docker Image"

Best Practices

  1. Error Handling: Implement robust error handling for network issues and job failures
  2. Resource Management: Properly manage threads, file handles, and network connections
  3. Security: Secure API tokens and credentials using environment variables or secret management
  4. Monitoring: Implement comprehensive logging and metrics collection
  5. Configuration: Make the agent highly configurable through environment variables
  6. Resilience: Implement retry logic for transient failures
  7. Cleanup: Properly clean up workspace and temporary files
  8. Scalability: Design for running multiple agents and concurrent job execution

Conclusion

This Java-based Buildkite agent provides:

  • Job Polling: Continuous polling for new jobs from Buildkite
  • Job Execution: Secure execution of build commands in isolated workspaces
  • Artifact Handling: Support for artifact uploads and downloads
  • Git Integration: Repository cloning and management
  • Health Monitoring: Comprehensive health checks and metrics
  • REST API: Management API for agent control and monitoring
  • Docker Support: Containerized deployment option

The agent can be extended with additional features like Docker-in-Docker support, custom plugins, and advanced logging capabilities to meet specific CI/CD requirements.

Pyroscope Profiling in Java
Explains how to use Pyroscope for continuous profiling in Java applications, helping developers analyze CPU and memory usage patterns to improve performance and identify bottlenecks.
https://macronepal.com/blog/pyroscope-profiling-in-java/

OpenTelemetry Metrics in Java: Comprehensive Guide
Provides a complete guide to collecting and exporting metrics in Java using OpenTelemetry, including counters, histograms, gauges, and integration with monitoring tools. (MACRO NEPAL)
https://macronepal.com/blog/opentelemetry-metrics-in-java-comprehensive-guide/

OTLP Exporter in Java: Complete Guide for OpenTelemetry
Explains how to configure OTLP exporters in Java to send telemetry data such as traces, metrics, and logs to monitoring systems using HTTP or gRPC protocols. (MACRO NEPAL)
https://macronepal.com/blog/otlp-exporter-in-java-complete-guide-for-opentelemetry/

Thanos Integration in Java: Global View of Metrics
Explains how to integrate Thanos with Java monitoring systems to create a scalable global metrics view across multiple Prometheus instances.

https://macronepal.com/blog/thanos-integration-in-java-global-view-of-metrics

Time Series with InfluxDB in Java: Complete Guide (Version 2)
Explains how to manage time-series data using InfluxDB in Java applications, including storing, querying, and analyzing metrics data.

https://macronepal.com/blog/time-series-with-influxdb-in-java-complete-guide-2

Time Series with InfluxDB in Java: Complete Guide
Provides an overview of integrating InfluxDB with Java for time-series data handling, including monitoring applications and managing performance metrics.

https://macronepal.com/blog/time-series-with-influxdb-in-java-complete-guide

Implementing Prometheus Remote Write in Java (Version 2)
Explains how to configure Java applications to send metrics data to Prometheus-compatible systems using the remote write feature for scalable monitoring.

https://macronepal.com/blog/implementing-prometheus-remote-write-in-java-a-complete-guide-2

Implementing Prometheus Remote Write in Java: Complete Guide
Provides instructions for sending metrics from Java services to Prometheus servers, enabling centralized monitoring and real-time analytics.

https://macronepal.com/blog/implementing-prometheus-remote-write-in-java-a-complete-guide

Building a TileServer GL in Java: Vector and Raster Tile Server
Explains how to build a TileServer GL in Java for serving vector and raster map tiles, useful for geographic visualization and mapping applications.

https://macronepal.com/blog/building-a-tileserver-gl-in-java-vector-and-raster-tile-server

Indoor Mapping in Java
Explains how to create indoor mapping systems in Java, including navigation inside buildings, spatial data handling, and visualization techniques.

Category : JAVA

Leave a Reply

Your email address will not be published. Required fields are marked *


Macro Nepal Helper