Skip to content

Commit

Permalink
Merge pull request #504 from zZHorizonZz/devservices
Browse files Browse the repository at this point in the history
feature(devservices): added bigtable dev service
  • Loading branch information
loicmathieu committed Sep 18, 2023
2 parents 519dbe8 + 744b3ec commit edb487e
Show file tree
Hide file tree
Showing 8 changed files with 305 additions and 36 deletions.
6 changes: 5 additions & 1 deletion bigtable/deployment/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
<groupId>io.quarkiverse.googlecloudservices</groupId>
<artifactId>quarkus-google-cloud-bigtable</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>gcloud</artifactId>
</dependency>
</dependencies>

<build>
Expand All @@ -46,4 +50,4 @@
</plugins>
</build>

</project>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import io.quarkus.deployment.builditem.FeatureBuildItem;

public class BigtableBuildSteps {
private static final String FEATURE = "google-cloud-bigtable";
protected static final String FEATURE = "google-cloud-bigtable";

@BuildStep
public FeatureBuildItem feature() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package io.quarkiverse.googlecloudservices.bigtable.deployment;

import io.quarkus.runtime.annotations.ConfigItem;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;

/**
* Root configuration class for Bigtable that operates at build time.
* This class provides a nested structure for configuration, including
* a separate group for the development service configuration.
*/
@ConfigRoot(name = "google.cloud.bigtable", phase = ConfigPhase.BUILD_TIME)
public class BigtableBuildTimeConfig {

/**
* Configuration for the Bigtable dev service.
* These settings will be used when Bigtable service is being configured
* for development purposes.
*/
@ConfigItem
public BigtableDevServiceConfig devservice;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package io.quarkiverse.googlecloudservices.bigtable.deployment;

import java.util.Optional;

import io.quarkus.runtime.annotations.ConfigGroup;
import io.quarkus.runtime.annotations.ConfigItem;

/**
* Configuration group for the Bigtable dev service. This class holds all the configuration properties
* related to the Google Cloud Bigtable service for development environments.
* <p>
* Here is an example of how to configure these properties:
* <p>
*
* <pre>
* quarkus.google.cloud.bigtable.deservice.enabled = true
* quarkus.google.cloud.bigtable.deservice.image-name = gcr.io/google.com/cloudsdktool/google-cloud-cli # optional
* quarkus.google.cloud.bigtable.deservice.emulatorPort = 9000 # optional
* </pre>
*/
@ConfigGroup
public class BigtableDevServiceConfig {

/**
* Indicates whether the Bigtable service should be enabled or not.
* The default value is 'false'.
*/
@ConfigItem(defaultValue = "false")
public boolean enabled;

/**
* Sets the Docker image name for the Google Cloud SDK.
* This image is used to emulate the Bigtable service in the development environment.
* The default value is 'gcr.io/google.com/cloudsdktool/google-cloud-cli'.
*/
@ConfigItem(defaultValue = "gcr.io/google.com/cloudsdktool/google-cloud-cli")
public String imageName;

/**
* Specifies the emulatorPort on which the Bigtable service should run in the development environment.
*/
@ConfigItem
public Optional<Integer> emulatorPort = Optional.empty();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
package io.quarkiverse.googlecloudservices.bigtable.deployment;

import java.time.Duration;
import java.util.List;
import java.util.Optional;

import org.jboss.logging.Logger;
import org.testcontainers.containers.BigtableEmulatorContainer;
import org.testcontainers.utility.DockerImageName;

import io.quarkus.deployment.IsNormal;
import io.quarkus.deployment.annotations.BuildStep;
import io.quarkus.deployment.annotations.BuildSteps;
import io.quarkus.deployment.builditem.CuratedApplicationShutdownBuildItem;
import io.quarkus.deployment.builditem.DevServicesResultBuildItem;
import io.quarkus.deployment.builditem.DevServicesSharedNetworkBuildItem;
import io.quarkus.deployment.builditem.DockerStatusBuildItem;
import io.quarkus.deployment.builditem.LaunchModeBuildItem;
import io.quarkus.deployment.console.ConsoleInstalledBuildItem;
import io.quarkus.deployment.console.StartupLogCompressor;
import io.quarkus.deployment.dev.devservices.GlobalDevServicesConfig;
import io.quarkus.deployment.logging.LoggingSetupBuildItem;

/**
* Processor responsible for managing Bigtable Dev Services.
* <p>
* The processor starts the Bigtable service in case it's not running.
*/
@BuildSteps(onlyIfNot = IsNormal.class, onlyIf = GlobalDevServicesConfig.Enabled.class)
public class BigtableDevServiceProcessor {

private static final Logger LOGGER = Logger.getLogger(BigtableDevServiceProcessor.class.getName());

// Running dev service instance
private static volatile DevServicesResultBuildItem.RunningDevService devService;
// Configuration for the Bigtable Dev service
private static volatile BigtableDevServiceConfig config;

@BuildStep
public DevServicesResultBuildItem start(DockerStatusBuildItem dockerStatusBuildItem,
BigtableBuildTimeConfig buildTimeConfig,
List<DevServicesSharedNetworkBuildItem> devServicesSharedNetworkBuildItem,
Optional<ConsoleInstalledBuildItem> consoleInstalledBuildItem,
CuratedApplicationShutdownBuildItem closeBuildItem,
LaunchModeBuildItem launchMode,
LoggingSetupBuildItem loggingSetupBuildItem,
GlobalDevServicesConfig globalDevServicesConfig) {
// If dev service is running and config has changed, stop the service
if (devService != null && !buildTimeConfig.devservice.equals(config)) {
stopContainer();
} else if (devService != null) {
return devService.toBuildItem();
}

// Set up log compressor for startup logs
StartupLogCompressor compressor = new StartupLogCompressor(
(launchMode.isTest() ? "(test) " : "") + "Google Cloud Bigtable Dev Services Starting:",
consoleInstalledBuildItem,
loggingSetupBuildItem);

// Try starting the container if conditions are met
try {
devService = startContainerIfAvailable(dockerStatusBuildItem, buildTimeConfig.devservice,
globalDevServicesConfig.timeout);
} catch (Throwable t) {
LOGGER.warn("Unable to start Bigtable dev service", t);
// Dump captured logs in case of an error
compressor.closeAndDumpCaptured();
return null;
} finally {
compressor.close();
}

return devService == null ? null : devService.toBuildItem();
}

/**
* Start the container if conditions are met.
*
* @param dockerStatusBuildItem, Docker status
* @param config, Configuration for the Bigtable service
* @param timeout, Optional timeout for starting the service
* @return Running service item, or null if the service couldn't be started
*/
private DevServicesResultBuildItem.RunningDevService startContainerIfAvailable(DockerStatusBuildItem dockerStatusBuildItem,
BigtableDevServiceConfig config,
Optional<Duration> timeout) {
if (!config.enabled) {
// Bigtable service explicitly disabled
LOGGER.debug("Not starting Dev Services for Bigtable as it has been disabled in the config");
return null;
}

if (!dockerStatusBuildItem.isDockerAvailable()) {
LOGGER.warn("Not starting devservice because docker is not available");
return null;
}

return startContainer(dockerStatusBuildItem, config, timeout);
}

/**
* Starts the Bigtable emulator container with provided configuration.
*
* @param dockerStatusBuildItem, Docker status
* @param config, Configuration for the Bigtable service
* @param timeout, Optional timeout for starting the service
* @return Running service item, or null if the service couldn't be started
*/
private DevServicesResultBuildItem.RunningDevService startContainer(DockerStatusBuildItem dockerStatusBuildItem,
BigtableDevServiceConfig config,
Optional<Duration> timeout) {
// Create and configure Bigtable emulator container
BigtableEmulatorContainer emulatorContainer = new QuarkusBigtableContainer(
DockerImageName.parse(config.imageName).asCompatibleSubstituteFor("gcr.io/google.com/cloudsdktool/cloud-sdk"),
config.emulatorPort.orElse(null));

// Set container startup timeout if provided
timeout.ifPresent(emulatorContainer::withStartupTimeout);
emulatorContainer.start();

// Set the config for the started container
BigtableDevServiceProcessor.config = config;

// Return running service item with container details
return new DevServicesResultBuildItem.RunningDevService(BigtableBuildSteps.FEATURE,
emulatorContainer.getContainerId(),
emulatorContainer::close, "quarkus.google.cloud.bigtable.emulator-host",
emulatorContainer.getEmulatorEndpoint());
}

/**
* Stops the running Bigtable emulator container.
*/
private void stopContainer() {
if (devService != null && devService.isOwner()) {
try {
// Try closing the running dev service
devService.close();
} catch (Throwable e) {
LOGGER.error("Failed to stop Bigtable container", e);
} finally {
devService = null;
}
}
}

/**
* Class for creating and configuring a Bigtable emulator container.
*/
private static class QuarkusBigtableContainer extends BigtableEmulatorContainer {

private final Integer fixedExposedPort;
private static final int INTERNAL_PORT = 9000;

private QuarkusBigtableContainer(DockerImageName dockerImageName, Integer fixedExposedPort) {
super(dockerImageName);
this.fixedExposedPort = fixedExposedPort;
}

/**
* Configures the Bigtable emulator container.
*/
@Override
public void configure() {
super.configure();

// Expose Bigtable emulatorPort
if (fixedExposedPort != null) {
addFixedExposedPort(fixedExposedPort, INTERNAL_PORT);
} else {
addExposedPort(INTERNAL_PORT);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,24 @@
import org.eclipse.microprofile.config.inject.ConfigProperty;

import com.google.api.gax.core.CredentialsProvider;
import com.google.api.gax.core.NoCredentialsProvider;
import com.google.api.gax.grpc.GrpcTransportChannel;
import com.google.api.gax.rpc.FixedTransportChannelProvider;
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
import com.google.cloud.bigtable.admin.v2.stub.BigtableTableAdminStubSettings;
import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
import com.google.cloud.bigtable.data.v2.models.Row;
import com.google.cloud.bigtable.data.v2.models.RowCell;
import com.google.cloud.bigtable.data.v2.models.RowMutation;

import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;

@Path("/bigtable")
public class BigtableResource {
private static final String INSTANCE_ID = "test-instance";
Expand All @@ -31,21 +40,47 @@ public class BigtableResource {
@ConfigProperty(name = "bigtable.authenticated", defaultValue = "true")
boolean authenticated;

@ConfigProperty(name = "quarkus.google.cloud.bigtable.emulator-host")
String emulatorHost;

@Inject
CredentialsProvider credentialsProvider;

@PostConstruct
void initBigtable() throws IOException {
BigtableTableAdminSettings.Builder settings = BigtableTableAdminSettings.newBuilder()
.setProjectId(projectId)
.setInstanceId(INSTANCE_ID);
if (authenticated) {
settings.setCredentialsProvider(credentialsProvider);
}
try (BigtableTableAdminClient adminClient = BigtableTableAdminClient.create(settings.build())) {
if (!adminClient.exists(TABLE_ID)) {
CreateTableRequest createTableRequest = CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_ID);
adminClient.createTable(createTableRequest);
if (emulatorHost != null) {
ManagedChannel channel = ManagedChannelBuilder.forTarget(emulatorHost).usePlaintext().build();

TransportChannelProvider channelProvider = FixedTransportChannelProvider.create(
GrpcTransportChannel.create(channel));
NoCredentialsProvider credentialsProvider = NoCredentialsProvider.create();

EnhancedBigtableTableAdminStub stub = EnhancedBigtableTableAdminStub.createEnhanced(
BigtableTableAdminStubSettings
.newBuilder()
.setTransportChannelProvider(channelProvider)
.setCredentialsProvider(credentialsProvider)
.build());

try (BigtableTableAdminClient adminClient = BigtableTableAdminClient.create(projectId, INSTANCE_ID, stub)) {
if (!adminClient.exists(TABLE_ID)) {
CreateTableRequest createTableRequest = CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_ID);
adminClient.createTable(createTableRequest);
}
}
} else {
BigtableTableAdminSettings.Builder settings = BigtableTableAdminSettings.newBuilder()
.setProjectId(projectId)
.setInstanceId(INSTANCE_ID);

if (authenticated) {
settings.setCredentialsProvider(credentialsProvider);
}
try (BigtableTableAdminClient adminClient = BigtableTableAdminClient.create(settings.build())) {
if (!adminClient.exists(TABLE_ID)) {
CreateTableRequest createTableRequest = CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_ID);
adminClient.createTable(createTableRequest);
}
}
}
}
Expand All @@ -55,6 +90,17 @@ public String bigtable() throws IOException {
BigtableDataSettings.Builder settings = BigtableDataSettings.newBuilder()
.setProjectId(projectId)
.setInstanceId(INSTANCE_ID);

if (emulatorHost != null) {
String[] hostAndPort = emulatorHost.split(":");
String host = hostAndPort[0];
int port = Integer.parseInt(hostAndPort[1]);

settings = BigtableDataSettings.newBuilderForEmulator(host, port)
.setProjectId(projectId)
.setInstanceId(INSTANCE_ID);
}

if (authenticated) {
settings.setCredentialsProvider(credentialsProvider);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
%test.pubsub.use-emulator=true
%test.quarkus.google.cloud.pubsub.devservice.enabled=true
%test.quarkus.google.cloud.firestore.devservice.enabled=true
%test.quarkus.google.cloud.bigtable.devservice.enabled=true

# Secret Manager Demo
# You can load secrets from Google Cloud Secret Manager with the ${sm//<SECRET_ID>} syntax.
Expand Down
Loading

0 comments on commit edb487e

Please sign in to comment.