1. Setup and Configuration
Maven Dependencies
<!-- pom.xml -->
<properties>
<google.cloud.version>26.20.0</google.cloud.version>
</properties>
<dependencies>
<!-- Core Cloud Libraries -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bom</artifactId>
<version>${google.cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Individual Services -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigquery</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-pubsub</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-firestore</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-spanner</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datastore</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-secretmanager</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-logging</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
</dependency>
</dependencies>
Authentication Setup
// GoogleCloudConfig.java
package com.gcp.config;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.ServiceOptions;
import java.io.FileInputStream;
import java.io.IOException;
public class GoogleCloudConfig {
// Method 1: Auto-discovery (recommended for GCP environments)
public static void setupDefaultCredentials() throws IOException {
// Uses GOOGLE_APPLICATION_CREDENTIALS environment variable
// Or metadata server in GCP environments
GoogleCredentials credentials = GoogleCredentials.getApplicationDefault();
}
// Method 2: Service account key file
public static GoogleCredentials setupWithServiceAccount(String keyPath) throws IOException {
GoogleCredentials credentials = GoogleCredentials.fromStream(new FileInputStream(keyPath));
return credentials;
}
// Method 3: Set project ID explicitly
public static void setProjectId(String projectId) {
System.setProperty("GOOGLE_CLOUD_PROJECT", projectId);
}
// Method 4: For local development with user credentials
public static GoogleCredentials setupWithUserCredentials() throws IOException {
GoogleCredentials credentials = GoogleCredentials.getApplicationDefault();
if (credentials.createScopedRequired()) {
credentials = credentials.createScoped("https://www.googleapis.com/auth/cloud-platform");
}
return credentials;
}
}
// CloudServiceFactory.java
package com.gcp.core;
import com.google.cloud.ServiceOptions;
import com.google.auth.Credentials;
public class CloudServiceFactory {
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
public static String getProjectId() {
return PROJECT_ID;
}
public static boolean isRunningOnGcp() {
return System.getenv("K_SERVICE") != null ||
System.getenv("GAE_INSTANCE") != null;
}
}
2. Cloud Storage
// CloudStorageService.java
package com.gcp.storage;
import com.google.cloud.storage.*;
import com.google.cloud.WriteChannel;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
public class CloudStorageService {
private final Storage storage;
private final String bucketName;
public CloudStorageService(Storage storage, String bucketName) {
this.storage = storage;
this.bucketName = bucketName;
}
public CloudStorageService(String bucketName) {
this(StorageOptions.getDefaultInstance().getService(), bucketName);
}
// Upload operations
public Blob uploadFile(String objectName, String filePath) throws IOException {
BlobId blobId = BlobId.of(bucketName, objectName);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId)
.setContentType("text/plain")
.setMetadata(ImmutableMap.of("uploaded-by", "java-client"))
.build();
return storage.create(blobInfo, Files.readAllBytes(Paths.get(filePath)));
}
public Blob uploadString(String objectName, String content) {
BlobId blobId = BlobId.of(bucketName, objectName);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
return storage.create(blobInfo, content.getBytes());
}
// Streaming upload for large files
public void uploadLargeFile(String objectName, String filePath) throws IOException {
BlobId blobId = BlobId.of(bucketName, objectName);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
try (WriteChannel writer = storage.writer(blobInfo)) {
byte[] buffer = new byte[1024 * 1024]; // 1MB buffer
try (FileInputStream input = new FileInputStream(filePath)) {
int limit;
while ((limit = input.read(buffer)) >= 0) {
writer.write(ByteBuffer.wrap(buffer, 0, limit));
}
}
}
}
// Download operations
public byte[] downloadFile(String objectName) {
Blob blob = storage.get(BlobId.of(bucketName, objectName));
return blob.getContent();
}
public String downloadAsString(String objectName) {
return new String(downloadFile(objectName));
}
public void downloadToFile(String objectName, String destPath) throws IOException {
Blob blob = storage.get(BlobId.of(bucketName, objectName));
blob.downloadTo(Paths.get(destPath));
}
// List objects
public List<Blob> listObjects(String prefix) {
List<Blob> blobs = new ArrayList<>();
Page<Blob> blobPage = storage.list(bucketName,
Storage.BlobListOption.prefix(prefix),
Storage.BlobListOption.pageSize(100));
for (Blob blob : blobPage.iterateAll()) {
blobs.add(blob);
}
return blobs;
}
// Generate signed URL
public String generateSignedUrl(String objectName, long durationMinutes) {
BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build();
URLSigner urlSigner = storage.signUrl(blobInfo, durationMinutes, TimeUnit.MINUTES,
Storage.SignUrlOption.withV4Signature());
return urlSigner.signUrl().toString();
}
// Delete operations
public boolean deleteObject(String objectName) {
return storage.delete(BlobId.of(bucketName, objectName));
}
// Copy object
public Blob copyObject(String sourceObject, String destObject) {
CopyWriter copyWriter = storage.copy(Storage.CopyRequest.newBuilder()
.setSource(BlobId.of(bucketName, sourceObject))
.setTarget(BlobId.of(bucketName, destObject))
.build());
return copyWriter.getResult();
}
// Bucket operations
public Bucket createBucket(String bucketName) {
return storage.create(BucketInfo.newBuilder(bucketName).build());
}
public void deleteBucket(String bucketName) {
storage.delete(bucketName);
}
public List<Bucket> listBuckets() {
List<Bucket> buckets = new ArrayList<>();
for (Bucket bucket : storage.list().iterateAll()) {
buckets.add(bucket);
}
return buckets;
}
}
// StorageAdvancedFeatures.java
package com.gcp.storage;
import com.google.cloud.storage.*;
public class StorageAdvancedFeatures {
private final Storage storage;
public StorageAdvancedFeatures(Storage storage) {
this.storage = storage;
}
// Set object lifecycle rules
public void setLifecycleRule(String bucketName) {
Bucket bucket = storage.get(bucketName);
Bucket updatedBucket = bucket.toBuilder()
.setLifecycleRules(List.of(
BucketInfo.LifecycleRule.newBuilder()
.setAction(BucketInfo.LifecycleRule.DeleteAction.newBuilder().build())
.setCondition(BucketInfo.LifecycleRule.Condition.newBuilder()
.setAge(30) // Delete after 30 days
.build())
.build()
))
.build()
.update();
}
// Enable versioning
public void enableVersioning(String bucketName) {
Bucket bucket = storage.get(bucketName);
bucket.toBuilder()
.setVersioningEnabled(true)
.build()
.update();
}
// Object retention
public void setRetentionPolicy(String bucketName, long retentionPeriodSeconds) {
Bucket bucket = storage.get(bucketName);
bucket.toBuilder()
.setRetentionPeriod(retentionPeriodSeconds)
.build()
.update();
}
// Get storage statistics
public StorageStats getStorageStats(String bucketName) {
long totalSize = 0;
long objectCount = 0;
for (Blob blob : storage.list(bucketName).iterateAll()) {
totalSize += blob.getSize();
objectCount++;
}
return new StorageStats(bucketName, objectCount, totalSize);
}
public static class StorageStats {
public final String bucketName;
public final long objectCount;
public final long totalSizeBytes;
public StorageStats(String bucketName, long objectCount, long totalSizeBytes) {
this.bucketName = bucketName;
this.objectCount = objectCount;
this.totalSizeBytes = totalSizeBytes;
}
public String getFormattedSize() {
if (totalSizeBytes < 1024) return totalSizeBytes + " B";
if (totalSizeBytes < 1024 * 1024) return (totalSizeBytes / 1024) + " KB";
if (totalSizeBytes < 1024 * 1024 * 1024) return (totalSizeBytes / (1024 * 1024)) + " MB";
return (totalSizeBytes / (1024 * 1024 * 1024)) + " GB";
}
}
}
3. BigQuery
// BigQueryService.java
package com.gcp.bigquery;
import com.google.cloud.bigquery.*;
import com.google.cloud.bigquery.TableResult;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
public class BigQueryService {
private final BigQuery bigquery;
private final String datasetId;
public BigQueryService(BigQuery bigquery, String datasetId) {
this.bigquery = bigquery;
this.datasetId = datasetId;
}
public BigQueryService(String datasetId) {
this(BigQueryOptions.getDefaultInstance().getService(), datasetId);
}
// Execute query
public TableResult executeQuery(String query) throws InterruptedException {
QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query).build();
JobId jobId = JobId.of(UUID.randomUUID().toString());
Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());
queryJob = queryJob.waitFor();
if (queryJob == null) {
throw new RuntimeException("Job no longer exists");
}
if (queryJob.getStatus().getError() != null) {
throw new RuntimeException(queryJob.getStatus().getError().toString());
}
return queryJob.getQueryResults();
}
// Create table
public Table createTable(String tableId, Schema schema) {
TableId table = TableId.of(datasetId, tableId);
TableDefinition tableDefinition = StandardTableDefinition.of(schema);
TableInfo tableInfo = TableInfo.newBuilder(table, tableDefinition).build();
return bigquery.create(tableInfo);
}
// Insert data
public void insertRows(String tableId, List<InsertAllRequest.RowToInsert> rows) {
TableId table = TableId.of(datasetId, tableId);
InsertAllRequest request = InsertAllRequest.newBuilder(table)
.setRows(rows)
.build();
InsertAllResponse response = bigquery.insertAll(request);
if (response.hasErrors()) {
System.err.println("Errors occurred while inserting rows:");
response.getInsertErrors().forEach((index, errors) -> {
System.err.printf("Row %d: %s%n", index, errors);
});
}
}
// Load data from Cloud Storage
public Job loadFromGcs(String tableId, String gcsUri, Schema schema) {
TableId table = TableId.of(datasetId, tableId);
LoadJobConfiguration loadConfig = LoadJobConfiguration.newBuilder(table, gcsUri)
.setSchema(schema)
.setFormatOptions(FormatOptions.json())
.build();
JobId jobId = JobId.of(UUID.randomUUID().toString());
Job loadJob = bigquery.create(JobInfo.newBuilder(loadConfig).setJobId(jobId).build());
try {
loadJob = loadJob.waitFor();
} catch (InterruptedException e) {
throw new RuntimeException("Job interrupted", e);
}
if (loadJob.getStatus().getError() != null) {
throw new RuntimeException(loadJob.getStatus().getError().toString());
}
return loadJob;
}
// Export to Cloud Storage
public Job exportToGcs(String tableId, String gcsUri) {
TableId table = TableId.of(datasetId, tableId);
ExtractJobConfiguration extractConfig = ExtractJobConfiguration.newBuilder(table, gcsUri)
.setFormat("JSON")
.build();
JobId jobId = JobId.of(UUID.randomUUID().toString());
Job extractJob = bigquery.create(JobInfo.newBuilder(extractConfig).setJobId(jobId).build());
try {
extractJob = extractJob.waitFor();
} catch (InterruptedException e) {
throw new RuntimeException("Job interrupted", e);
}
return extractJob;
}
// Get table info
public Table getTable(String tableId) {
return bigquery.getTable(datasetId, tableId);
}
// Delete table
public boolean deleteTable(String tableId) {
return bigquery.delete(TableId.of(datasetId, tableId));
}
// List tables in dataset
public List<Table> listTables() {
List<Table> tables = new ArrayList<>();
for (Table table : bigquery.listTables(datasetId).iterateAll()) {
tables.add(table);
}
return tables;
}
}
// BigQuerySchemaExamples.java
package com.gcp.bigquery;
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardSQLTypeName;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FieldList;
public class BigQuerySchemaExamples {
public static Schema createUserSchema() {
return Schema.of(
Field.newBuilder("user_id", StandardSQLTypeName.STRING)
.setMode(Field.Mode.REQUIRED)
.build(),
Field.newBuilder("email", StandardSQLTypeName.STRING)
.setMode(Field.Mode.REQUIRED)
.build(),
Field.newBuilder("name", StandardSQLTypeName.STRING)
.setMode(Field.Mode.NULLABLE)
.build(),
Field.newBuilder("age", StandardSQLTypeName.INT64)
.setMode(Field.Mode.NULLABLE)
.build(),
Field.newBuilder("created_at", StandardSQLTypeName.TIMESTAMP)
.setMode(Field.Mode.REQUIRED)
.build(),
Field.newBuilder("preferences", StandardSQLTypeName.JSON)
.setMode(Field.Mode.NULLABLE)
.build()
);
}
public static Schema createNestedSchema() {
Field address = Field.newBuilder("address", StandardSQLTypeName.STRUCT,
Field.of("street", StandardSQLTypeName.STRING),
Field.of("city", StandardSQLTypeName.STRING),
Field.of("state", StandardSQLTypeName.STRING),
Field.of("zip_code", StandardSQLTypeName.STRING)
).build();
Field orders = Field.newBuilder("orders", StandardSQLTypeName.STRUCT,
Field.of("order_id", StandardSQLTypeName.STRING),
Field.of("amount", StandardSQLTypeName.FLOAT64),
Field.of("items", StandardSQLTypeName.STRING)
).setMode(Field.Mode.REPEATED).build();
return Schema.of(
Field.of("customer_id", StandardSQLTypeName.STRING),
address,
orders
);
}
}
// BigQueryAnalytics.java
package com.gcp.bigquery;
import com.google.cloud.bigquery.*;
import java.util.Map;
public class BigQueryAnalytics {
private final BigQuery bigquery;
public BigQueryAnalytics(BigQuery bigquery) {
this.bigquery = bigquery;
}
// Run analytical queries
public TableResult calculateUserMetrics(String datasetId, String tableId) throws InterruptedException {
String query = String.format(
"SELECT " +
" COUNT(*) as total_users, " +
" AVG(age) as average_age, " +
" COUNTIF(age >= 18 AND age <= 35) as young_adults, " +
" MAX(created_at) as latest_signup " +
"FROM `%s.%s.users`",
datasetId, tableId
);
return executeQuery(query);
}
// Time-series analysis
public TableResult timeSeriesAnalysis(String datasetId, String eventsTable) throws InterruptedException {
String query = String.format(
"SELECT " +
" DATE(timestamp) as date, " +
" COUNT(*) as event_count, " +
" COUNT(DISTINCT user_id) as unique_users, " +
" AVG(duration) as avg_duration " +
"FROM `%s.%s.events` " +
"WHERE timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY) " +
"GROUP BY date " +
"ORDER BY date DESC",
datasetId, eventsTable
);
return executeQuery(query);
}
// Cohort analysis
public TableResult cohortAnalysis(String datasetId, String usersTable, String ordersTable) throws InterruptedException {
String query = String.format(
"WITH user_cohorts AS ( " +
" SELECT " +
" user_id, " +
" DATE_TRUNC(DATE(created_at), MONTH) as signup_month " +
" FROM `%s.%s` " +
"), " +
"order_metrics AS ( " +
" SELECT " +
" user_id, " +
" DATE_TRUNC(DATE(order_date), MONTH) as order_month, " +
" COUNT(*) as order_count, " +
" SUM(amount) as total_amount " +
" FROM `%s.%s` " +
" GROUP BY user_id, order_month " +
") " +
"SELECT " +
" uc.signup_month, " +
" DATE_DIFF(om.order_month, uc.signup_month, MONTH) as months_since_signup, " +
" COUNT(DISTINCT uc.user_id) as active_users, " +
" AVG(om.order_count) as avg_orders, " +
" AVG(om.total_amount) as avg_revenue " +
"FROM user_cohorts uc " +
"JOIN order_metrics om ON uc.user_id = om.user_id " +
"GROUP BY signup_month, months_since_signup " +
"ORDER BY signup_month, months_since_signup",
datasetId, usersTable, datasetId, ordersTable
);
return executeQuery(query);
}
private TableResult executeQuery(String query) throws InterruptedException {
QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query).build();
return bigquery.query(queryConfig);
}
}
4. Pub/Sub
// PubSubService.java
package com.gcp.pubsub;
import com.google.cloud.pubsub.v1.*;
import com.google.protobuf.ByteString;
import com.google.pubsub.v1.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
public class PubSubService {
private final TopicAdminClient topicAdmin;
private final SubscriptionAdminClient subscriptionAdmin;
private final String projectId;
public PubSubService(String projectId) throws IOException {
this.topicAdmin = TopicAdminClient.create();
this.subscriptionAdmin = SubscriptionAdminClient.create();
this.projectId = projectId;
}
// Topic management
public Topic createTopic(String topicId) {
TopicName topicName = TopicName.of(projectId, topicId);
return topicAdmin.createTopic(topicName);
}
public void deleteTopic(String topicId) {
TopicName topicName = TopicName.of(projectId, topicId);
topicAdmin.deleteTopic(topicName);
}
public List<Topic> listTopics() {
List<Topic> topics = new ArrayList<>();
TopicAdminClient.ListTopicsPagedResponse response = topicAdmin.listTopics(ProjectName.of(projectId));
for (Topic topic : response.iterateAll()) {
topics.add(topic);
}
return topics;
}
// Subscription management
public Subscription createSubscription(String topicId, String subscriptionId) {
TopicName topicName = TopicName.of(projectId, topicId);
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
return subscriptionAdmin.createSubscription(
subscriptionName, topicName, PushConfig.getDefaultInstance(), 10
);
}
public void deleteSubscription(String subscriptionId) {
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
subscriptionAdmin.deleteSubscription(subscriptionName);
}
// Publishing messages
public String publishMessage(String topicId, String message) throws ExecutionException, InterruptedException {
return publishMessage(topicId, message, null);
}
public String publishMessage(String topicId, String message, Map<String, String> attributes)
throws ExecutionException, InterruptedException {
try (Publisher publisher = Publisher.newBuilder(TopicName.of(projectId, topicId)).build()) {
PubsubMessage pubsubMessage = PubsubMessage.newBuilder()
.setData(ByteString.copyFromUtf8(message))
.putAllAttributes(attributes != null ? attributes : Map.of())
.build();
ApiFuture<String> future = publisher.publish(pubsubMessage);
return future.get();
} catch (IOException e) {
throw new RuntimeException("Failed to create publisher", e);
}
}
// Batch publishing
public List<String> publishMessages(String topicId, List<String> messages)
throws ExecutionException, InterruptedException {
try (Publisher publisher = Publisher.newBuilder(TopicName.of(projectId, topicId)).build()) {
List<ApiFuture<String>> futures = new ArrayList<>();
for (String message : messages) {
PubsubMessage pubsubMessage = PubsubMessage.newBuilder()
.setData(ByteString.copyFromUtf8(message))
.build();
futures.add(publisher.publish(pubsubMessage));
}
List<String> messageIds = new ArrayList<>();
for (ApiFuture<String> future : futures) {
messageIds.add(future.get());
}
return messageIds;
} catch (IOException e) {
throw new RuntimeException("Failed to create publisher", e);
}
}
// Synchronous message pulling
public List<ReceivedMessage> pullMessages(String subscriptionId, int maxMessages) {
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
try (Subscriber subscriber = Subscriber.newBuilder(subscriptionName,
(message, consumer) -> {
// Message handler - will be used in async mode
consumer.ack();
}).build()) {
PullRequest pullRequest = PullRequest.newBuilder()
.setSubscription(subscriptionName.toString())
.setMaxMessages(maxMessages)
.build();
PullResponse pullResponse = subscriber.getStub().pullCallable().call(pullRequest);
return pullResponse.getReceivedMessagesList();
}
}
// Asynchronous message processing
public void startAsyncSubscriber(String subscriptionId, MessageProcessor processor) {
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
MessageReceiver receiver = (message, consumer) -> {
try {
processor.processMessage(message);
consumer.ack();
} catch (Exception e) {
System.err.println("Error processing message: " + e.getMessage());
consumer.nack();
}
};
Subscriber subscriber = Subscriber.newBuilder(subscriptionName, receiver).build();
subscriber.startAsync().awaitRunning();
System.out.println("Started async subscriber for: " + subscriptionId);
}
public interface MessageProcessor {
void processMessage(PubsubMessage message);
}
public void close() {
topicAdmin.close();
subscriptionAdmin.close();
}
}
// PubSubAdvancedFeatures.java
package com.gcp.pubsub;
import com.google.cloud.pubsub.v1.*;
import com.google.pubsub.v1.*;
import java.util.Map;
public class PubSubAdvancedFeatures {
private final String projectId;
public PubSubAdvancedFeatures(String projectId) {
this.projectId = projectId;
}
// Create subscription with dead letter policy
public Subscription createSubscriptionWithDLQ(String topicId, String subscriptionId,
String deadLetterTopicId) {
TopicName topicName = TopicName.of(projectId, topicId);
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
TopicName deadLetterTopicName = TopicName.of(projectId, deadLetterTopicId);
Subscription.Builder subscriptionBuilder = Subscription.newBuilder()
.setName(subscriptionName.toString())
.setTopic(topicName.toString())
.setAckDeadlineSeconds(30);
// Set dead letter policy
subscriptionBuilder.setDeadLetterPolicy(DeadLetterPolicy.newBuilder()
.setDeadLetterTopic(deadLetterTopicName.toString())
.setMaxDeliveryAttempts(5)
.build());
try (SubscriptionAdminClient subscriptionAdmin = SubscriptionAdminClient.create()) {
return subscriptionAdmin.createSubscription(subscriptionBuilder.build());
} catch (Exception e) {
throw new RuntimeException("Failed to create subscription with DLQ", e);
}
}
// Create subscription with filtering
public Subscription createFilteredSubscription(String topicId, String subscriptionId,
String filter) {
TopicName topicName = TopicName.of(projectId, topicId);
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
Subscription subscription = Subscription.newBuilder()
.setName(subscriptionName.toString())
.setTopic(topicName.toString())
.setFilter(filter)
.build();
try (SubscriptionAdminClient subscriptionAdmin = SubscriptionAdminClient.create()) {
return subscriptionAdmin.createSubscription(subscription);
} catch (Exception e) {
throw new RuntimeException("Failed to create filtered subscription", e);
}
}
// Configure retry policy
public void configureRetryPolicy(String subscriptionId, int minBackoffSeconds,
int maxBackoffSeconds) {
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
try (SubscriptionAdminClient subscriptionAdmin = SubscriptionAdminClient.create()) {
Subscription subscription = subscriptionAdmin.getSubscription(subscriptionName);
Subscription updatedSubscription = subscription.toBuilder()
.setRetryPolicy(RetryPolicy.newBuilder()
.setMinimumBackoff(com.google.protobuf.Duration.newBuilder()
.setSeconds(minBackoffSeconds)
.build())
.setMaximumBackoff(com.google.protobuf.Duration.newBuilder()
.setSeconds(maxBackoffSeconds)
.build())
.build())
.build();
subscriptionAdmin.updateSubscription(updatedSubscription);
} catch (Exception e) {
throw new RuntimeException("Failed to configure retry policy", e);
}
}
// Get subscription stats
public SubscriptionStats getSubscriptionStats(String subscriptionId) {
SubscriptionName subscriptionName = SubscriptionName.of(projectId, subscriptionId);
try (SubscriptionAdminClient subscriptionAdmin = SubscriptionAdminClient.create()) {
Subscription subscription = subscriptionAdmin.getSubscription(subscriptionName);
// In production, you'd use Cloud Monitoring to get actual metrics
return new SubscriptionStats(
subscription.getName(),
0, // num_undelivered_messages - would come from monitoring
0, // oldest_unacked_message_age - would come from monitoring
subscription.getAckDeadlineSeconds()
);
} catch (Exception e) {
throw new RuntimeException("Failed to get subscription stats", e);
}
}
public static class SubscriptionStats {
public final String subscriptionName;
public final long undeliveredMessages;
public final long oldestUnackedMessageAge;
public final int ackDeadlineSeconds;
public SubscriptionStats(String subscriptionName, long undeliveredMessages,
long oldestUnackedMessageAge, int ackDeadlineSeconds) {
this.subscriptionName = subscriptionName;
this.undeliveredMessages = undeliveredMessages;
this.oldestUnackedMessageAge = oldestUnackedMessageAge;
this.ackDeadlineSeconds = ackDeadlineSeconds;
}
}
}
5. Firestore
// FirestoreService.java
package com.gcp.firestore;
import com.google.cloud.firestore.*;
import com.google.api.core.ApiFuture;
import java.util.*;
import java.util.concurrent.ExecutionException;
public class FirestoreService {
private final Firestore firestore;
public FirestoreService(Firestore firestore) {
this.firestore = firestore;
}
public FirestoreService() {
this(FirestoreOptions.getDefaultInstance().getService());
}
// Create document
public WriteResult createDocument(String collection, String documentId, Map<String, Object> data)
throws ExecutionException, InterruptedException {
DocumentReference docRef = firestore.collection(collection).document(documentId);
ApiFuture<WriteResult> result = docRef.set(data);
return result.get();
}
// Create document with auto-generated ID
public String createDocumentAutoId(String collection, Map<String, Object> data)
throws ExecutionException, InterruptedException {
DocumentReference docRef = firestore.collection(collection).document();
ApiFuture<WriteResult> result = docRef.set(data);
result.get();
return docRef.getId();
}
// Read document
public DocumentSnapshot readDocument(String collection, String documentId)
throws ExecutionException, InterruptedException {
DocumentReference docRef = firestore.collection(collection).document(documentId);
ApiFuture<DocumentSnapshot> future = docRef.get();
return future.get();
}
// Update document
public WriteResult updateDocument(String collection, String documentId, Map<String, Object> updates)
throws ExecutionException, InterruptedException {
DocumentReference docRef = firestore.collection(collection).document(documentId);
ApiFuture<WriteResult> result = docRef.update(updates);
return result.get();
}
// Delete document
public WriteResult deleteDocument(String collection, String documentId)
throws ExecutionException, InterruptedException {
DocumentReference docRef = firestore.collection(collection).document(documentId);
ApiFuture<WriteResult> result = docRef.delete();
return result.get();
}
// Query documents
public List<DocumentSnapshot> queryDocuments(String collection, String field, Object value)
throws ExecutionException, InterruptedException {
ApiFuture<QuerySnapshot> future = firestore.collection(collection)
.whereEqualTo(field, value)
.get();
QuerySnapshot querySnapshot = future.get();
return querySnapshot.getDocuments();
}
// Complex query
public List<DocumentSnapshot> queryWithConditions(String collection,
Map<String, Object> conditions)
throws ExecutionException, InterruptedException {
Query query = firestore.collection(collection);
for (Map.Entry<String, Object> condition : conditions.entrySet()) {
query = query.whereEqualTo(condition.getKey(), condition.getValue());
}
ApiFuture<QuerySnapshot> future = query.get();
QuerySnapshot querySnapshot = future.get();
return querySnapshot.getDocuments();
}
// Batch write
public List<WriteResult> batchWrite(List<WriteOperation> operations)
throws ExecutionException, InterruptedException {
WriteBatch batch = firestore.batch();
for (WriteOperation operation : operations) {
DocumentReference docRef = firestore.collection(operation.collection)
.document(operation.documentId);
switch (operation.operationType) {
case SET:
batch.set(docRef, operation.data);
break;
case UPDATE:
batch.update(docRef, operation.data);
break;
case DELETE:
batch.delete(docRef);
break;
}
}
ApiFuture<List<WriteResult>> result = batch.commit();
return result.get();
}
// Transaction
public String runTransaction(String collection, String documentId, double amount)
throws ExecutionException, InterruptedException {
ApiFuture<String> future = firestore.runTransaction(transaction -> {
DocumentReference docRef = firestore.collection(collection).document(documentId);
DocumentSnapshot snapshot = transaction.get(docRef).get();
if (!snapshot.exists()) {
throw new RuntimeException("Document does not exist");
}
Double currentBalance = snapshot.getDouble("balance");
if (currentBalance == null) {
throw new RuntimeException("Balance field not found");
}
double newBalance = currentBalance + amount;
if (newBalance < 0) {
throw new RuntimeException("Insufficient funds");
}
transaction.update(docRef, "balance", newBalance);
return String.format("Updated balance from %.2f to %.2f", currentBalance, newBalance);
});
return future.get();
}
// Real-time listener
public void addCollectionListener(String collection, DocumentChangeListener listener) {
firestore.collection(collection).addSnapshotListener(listener);
}
public interface DocumentChangeListener {
void onEvent(QuerySnapshot snapshot, FirestoreException e);
}
public static class WriteOperation {
public enum OperationType { SET, UPDATE, DELETE }
public final String collection;
public final String documentId;
public final Map<String, Object> data;
public final OperationType operationType;
public WriteOperation(String collection, String documentId,
Map<String, Object> data, OperationType operationType) {
this.collection = collection;
this.documentId = documentId;
this.data = data;
this.operationType = operationType;
}
}
}
// FirestoreDataModels.java
package com.gcp.firestore;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
public class FirestoreDataModels {
public static Map<String, Object> createUserDocument(String userId, String email,
String name, int age) {
Map<String, Object> user = new HashMap<>();
user.put("userId", userId);
user.put("email", email);
user.put("name", name);
user.put("age", age);
user.put("createdAt", new Date());
user.put("updatedAt", new Date());
user.put("active", true);
// Nested object
Map<String, Object> preferences = new HashMap<>();
preferences.put("theme", "dark");
preferences.put("notifications", true);
preferences.put("language", "en");
user.put("preferences", preferences);
return user;
}
public static Map<String, Object> createOrderDocument(String orderId, String userId,
List<OrderItem> items, double total) {
Map<String, Object> order = new HashMap<>();
order.put("orderId", orderId);
order.put("userId", userId);
order.put("items", convertItemsToMap(items));
order.put("total", total);
order.put("status", "pending");
order.put("createdAt", new Date());
// Address information
Map<String, Object> shippingAddress = new HashMap<>();
shippingAddress.put("street", "123 Main St");
shippingAddress.put("city", "San Francisco");
shippingAddress.put("state", "CA");
shippingAddress.put("zipCode", "94105");
order.put("shippingAddress", shippingAddress);
return order;
}
private static List<Map<String, Object>> convertItemsToMap(List<OrderItem> items) {
List<Map<String, Object>> itemList = new ArrayList<>();
for (OrderItem item : items) {
Map<String, Object> itemMap = new HashMap<>();
itemMap.put("productId", item.productId);
itemMap.put("name", item.name);
itemMap.put("quantity", item.quantity);
itemMap.put("price", item.price);
itemList.add(itemMap);
}
return itemList;
}
public static class OrderItem {
public final String productId;
public final String name;
public final int quantity;
public final double price;
public OrderItem(String productId, String name, int quantity, double price) {
this.productId = productId;
this.name = name;
this.quantity = quantity;
this.price = price;
}
}
}
6. Secret Manager
// SecretManagerService.java
package com.gcp.secrets;
import com.google.cloud.secretmanager.v1.*;
import com.google.protobuf.ByteString;
import java.io.IOException;
public class SecretManagerService {
private final SecretManagerServiceClient client;
private final String projectId;
public SecretManagerService(String projectId) throws IOException {
this.client = SecretManagerServiceClient.create();
this.projectId = projectId;
}
// Create secret
public Secret createSecret(String secretId) {
ProjectName projectName = ProjectName.of(projectId);
Secret secret = Secret.newBuilder().setReplication(Replication.newBuilder()
.setAutomatic(Replication.Automatic.newBuilder().build())
.build()).build();
return client.createSecret(projectName, secretId, secret);
}
// Add secret version
public SecretVersion addSecretVersion(String secretId, String payload) {
SecretName secretName = SecretName.of(projectId, secretId);
SecretPayload secretPayload = SecretPayload.newBuilder()
.setData(ByteString.copyFromUtf8(payload))
.build();
return client.addSecretVersion(secretName, secretPayload);
}
// Access secret version
public String accessSecret(String secretId, String version) {
SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretId, version);
AccessSecretVersionResponse response = client.accessSecretVersion(secretVersionName);
return response.getPayload().getData().toStringUtf8();
}
// Access latest secret version
public String accessLatestSecret(String secretId) {
return accessSecret(secretId, "latest");
}
// List secrets
public List<Secret> listSecrets() {
ProjectName projectName = ProjectName.of(projectId);
List<Secret> secrets = new ArrayList<>();
for (Secret secret : client.listSecrets(projectName).iterateAll()) {
secrets.add(secret);
}
return secrets;
}
// Delete secret
public void deleteSecret(String secretId) {
SecretName secretName = SecretName.of(projectId, secretId);
client.deleteSecret(secretName);
}
// Enable secret version
public SecretVersion enableSecretVersion(String secretId, String version) {
SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretId, version);
return client.enableSecretVersion(secretVersionName);
}
// Disable secret version
public SecretVersion disableSecretVersion(String secretId, String version) {
SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretId, version);
return client.disableSecretVersion(secretVersionName);
}
// Destroy secret version
public SecretVersion destroySecretVersion(String secretId, String version) {
SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretId, version);
return client.destroySecretVersion(secretVersionName);
}
public void close() {
client.close();
}
}
7. Comprehensive Example Application
// CloudNativeApplication.java
package com.gcp.app;
import com.gcp.bigquery.BigQueryService;
import com.gcp.firestore.FirestoreService;
import com.gcp.pubsub.PubSubService;
import com.gcp.storage.CloudStorageService;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
public class CloudNativeApplication {
private final PubSubService pubSubService;
private final FirestoreService firestoreService;
private final BigQueryService bigQueryService;
private final CloudStorageService storageService;
public CloudNativeApplication(String projectId, String datasetId, String bucketName)
throws Exception {
this.pubSubService = new PubSubService(projectId);
this.firestoreService = new FirestoreService();
this.bigQueryService = new BigQueryService(datasetId);
this.storageService = new CloudStorageService(bucketName);
}
// Process user registration event
public void processUserRegistration(String userId, String email, String name, int age)
throws ExecutionException, InterruptedException {
// 1. Store user in Firestore
Map<String, Object> userData = new HashMap<>();
userData.put("userId", userId);
userData.put("email", email);
userData.put("name", name);
userData.put("age", age);
userData.put("registeredAt", System.currentTimeMillis());
userData.put("status", "active");
firestoreService.createDocument("users", userId, userData);
// 2. Publish registration event to Pub/Sub
Map<String, String> attributes = new HashMap<>();
attributes.put("event_type", "user_registration");
attributes.put("user_id", userId);
String message = String.format("User %s registered with email %s", name, email);
pubSubService.publishMessage("user-events", message, attributes);
// 3. Store registration log in Cloud Storage
String logEntry = String.format("%s,%s,%s,%d,%d%n",
userId, email, name, age, System.currentTimeMillis());
storageService.uploadString("user-registrations/" + userId + ".log", logEntry);
System.out.println("Processed user registration for: " + email);
}
// Analyze user data
public void analyzeUserData() throws InterruptedException {
String query = "SELECT " +
" COUNT(*) as total_users, " +
" AVG(age) as average_age, " +
" COUNTIF(age >= 18 AND age <= 35) as young_adults " +
"FROM `users`";
var result = bigQueryService.executeQuery(query);
result.iterateAll().forEach(row -> {
System.out.printf("Total Users: %d%n", row.get("total_users").getLongValue());
System.out.printf("Average Age: %.1f%n", row.get("average_age").getDoubleValue());
System.out.printf("Young Adults: %d%n", row.get("young_adults").getLongValue());
});
}
// Batch process users
public void batchProcessUsers(List<String> userIds) throws ExecutionException, InterruptedException {
List<PubSubService.WriteOperation> operations = new ArrayList<>();
for (String userId : userIds) {
Map<String, Object> update = new HashMap<>();
update.put("processed", true);
update.put("processedAt", System.currentTimeMillis());
operations.add(new PubSubService.WriteOperation(
"users", userId, update, PubSubService.WriteOperation.OperationType.UPDATE
));
}
firestoreService.batchWrite(operations);
System.out.println("Batch processed " + userIds.size() + " users");
}
public static void main(String[] args) throws Exception {
String projectId = "your-project-id";
String datasetId = "user_analytics";
String bucketName = "user-data-bucket";
CloudNativeApplication app = new CloudNativeApplication(projectId, datasetId, bucketName);
// Example usage
app.processUserRegistration("user123", "[email protected]", "John Doe", 30);
app.analyzeUserData();
// Start async message processing
app.pubSubService.startAsyncSubscriber("user-registrations", message -> {
String data = message.getData().toStringUtf8();
System.out.println("Received message: " + data);
// Process the message
Map<String, String> attributes = message.getAttributesMap();
String eventType = attributes.get("event_type");
String userId = attributes.get("user_id");
System.out.printf("Event: %s, User: %s%n", eventType, userId);
});
}
}
Key Features Covered:
- Authentication & Configuration: Multiple credential methods
- Cloud Storage: File operations, signed URLs, lifecycle management
- BigQuery: Query execution, data loading, analytical queries
- Pub/Sub: Message publishing, async processing, dead letter queues
- Firestore: Document operations, queries, transactions, real-time listeners
- Secret Manager: Secure secret storage and retrieval
- Integration: Complete cloud-native application example
This comprehensive implementation provides production-ready code for working with Google Cloud services in Java, following best practices for error handling, resource management, and performance optimization.