From 41b204d83fa29ff224d58373f6c82a0cd71a9660 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 31 Jul 2017 17:47:32 -0700 Subject: [PATCH 1/2] Migrate to the Azure Storage SDK --- breakerbox-azure/pom.xml | 4 +- .../yammer/breakerbox/azure/AzureStore.java | 28 ++++---- .../azure/AzureTableConfiguration.java | 2 +- .../yammer/breakerbox/azure/TableClient.java | 67 +++++++++---------- .../breakerbox/azure/TableClientFactory.java | 19 +++--- .../breakerbox/azure/core/TableKey.java | 2 +- .../breakerbox/azure/core/TableType.java | 2 +- .../azure/model/DependencyEntity.java | 2 +- .../breakerbox/azure/model/ServiceEntity.java | 2 +- .../model/tests/DependencyEntityTest.java | 2 +- pom.xml | 22 ++---- 11 files changed, 71 insertions(+), 81 deletions(-) diff --git a/breakerbox-azure/pom.xml b/breakerbox-azure/pom.xml index 27f6f5b..1760642 100644 --- a/breakerbox-azure/pom.xml +++ b/breakerbox-azure/pom.xml @@ -15,8 +15,8 @@ breakerbox-store - com.microsoft.windowsazure - microsoft-windowsazure-api + com.microsoft.azure + azure-storage io.dropwizard diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureStore.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureStore.java index 1238369..5085a43 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureStore.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureStore.java @@ -5,8 +5,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; -import com.microsoft.windowsazure.services.table.client.TableConstants; -import com.microsoft.windowsazure.services.table.client.TableQuery; +import com.microsoft.azure.storage.table.TableQuery; import com.yammer.breakerbox.azure.core.TableId; import com.yammer.breakerbox.azure.core.TableType; import com.yammer.breakerbox.azure.healthchecks.TableClientHealthcheck; @@ -23,6 +22,8 @@ import org.joda.time.DateTime; public class AzureStore extends BreakerboxStore { + public static final String PARTITION_KEY = "PartitionKey"; + public static final String ROW_KEY = "RowKey"; protected final TableClient tableClient; public AzureStore(AzureTableConfiguration azureTableConfiguration, @@ -102,8 +103,9 @@ public Iterable listDependenciesFor(final ServiceId serviceId) { @Override public Iterable allDependenciesFor(DependencyId dependencyId, ServiceId serviceId) { try (Timer.Context timerContext = dependencyConfigs.time()) { - return Entities.toDependencyModelList(tableClient.search(TableQuery - .from(TableId.DEPENDENCY.toString(), DependencyEntity.class) + return Entities.toDependencyModelList(tableClient.search( + TableId.DEPENDENCY, TableQuery + .from(DependencyEntity.class) .where(TableQuery.combineFilters( partitionEquals(dependencyId), TableQuery.Operators.AND, @@ -134,8 +136,8 @@ private Optional fetchByTimestamp(DependencyId dependencyId, l private ImmutableList allServiceEntities(ServiceId serviceId) { try (Timer.Context timerContext = listService.time()) { - return tableClient.search(TableQuery - .from(TableId.SERVICE.toString(), ServiceEntity.class) + return tableClient.search(TableId.SERVICE, TableQuery + .from(ServiceEntity.class) .where(partitionKeyEquals(serviceId))); } } @@ -143,22 +145,22 @@ private ImmutableList allServiceEntities(ServiceId serviceId) { private static String partitionKeyEquals(ServiceId serviceId) { return TableQuery .generateFilterCondition( - TableConstants.PARTITION_KEY, + PARTITION_KEY, TableQuery.QueryComparisons.EQUAL, serviceId.getId()); } private ImmutableList allServiceEntities() { try (Timer.Context timerContext = listService.time()) { - return tableClient.search(TableQuery - .from(TableId.SERVICE.toString(), ServiceEntity.class)); + return tableClient.search(TableId.SERVICE, TableQuery + .from(ServiceEntity.class)); } } private ImmutableList getConfiguration(DependencyId dependencyId, long targetTimeStamp) { try (Timer.Context timerContext = dependencyConfigs.time()) { - return tableClient.search(TableQuery - .from(TableId.DEPENDENCY.toString(), DependencyEntity.class) + return tableClient.search(TableId.DEPENDENCY, TableQuery + .from(DependencyEntity.class) .where(TableQuery.combineFilters( partitionEquals(dependencyId), TableQuery.Operators.AND, @@ -175,7 +177,7 @@ private static String serviceIdEquals(ServiceId serviceId) { private static String timestampEquals(long timestamp) { return TableQuery.generateFilterCondition( - TableConstants.ROW_KEY, + ROW_KEY, TableQuery.QueryComparisons.EQUAL, String.valueOf(timestamp) ); @@ -183,7 +185,7 @@ private static String timestampEquals(long timestamp) { private static String partitionEquals(DependencyId dependencyId) { return TableQuery.generateFilterCondition( - TableConstants.PARTITION_KEY, + PARTITION_KEY, TableQuery.QueryComparisons.EQUAL, dependencyId.getId()); } diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureTableConfiguration.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureTableConfiguration.java index a5dd23d..a7bb9bd 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureTableConfiguration.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/AzureTableConfiguration.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.windowsazure.services.core.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import io.dropwizard.util.Duration; import javax.validation.Valid; diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClient.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClient.java index 2fb0361..78908c9 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClient.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClient.java @@ -2,8 +2,8 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; -import com.microsoft.windowsazure.services.core.storage.StorageException; -import com.microsoft.windowsazure.services.table.client.*; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.table.*; import com.yammer.breakerbox.azure.core.AzureTableName; import com.yammer.breakerbox.azure.core.TableKey; import com.yammer.breakerbox.azure.core.TableType; @@ -24,13 +24,21 @@ public TableClient(CloudTableClient cloudTableClient) { this.cloudTableClient = checkNotNull(cloudTableClient, "cloudTableClient cannot be null"); } + private CloudTable tableRefrence(AzureTableName table) { + try { + return cloudTableClient.getTableReference(table.toString()); + } catch (URISyntaxException e) { + LOG.warn("URI exception creating table: {},", table, e); + } catch (StorageException e) { + LOG.warn("Error generating TableClient: {}", table, e); + } + throw new IllegalStateException("Could not create table: " + table); + } + public void create(AzureTableName table) { try { - final CloudTable cloudTable = new CloudTable(table.toString(), cloudTableClient); - cloudTable.createIfNotExist(); + tableRefrence(table).createIfNotExists(); return; - } catch (URISyntaxException e) { - LOG.warn("URI exception creating table", e); } catch (StorageException e) { LOG.warn("Error creating table: {}", table, e); } @@ -39,9 +47,9 @@ public void create(AzureTableName table) { public boolean insert(TableType entity) { try { - final TableResult tableResult = cloudTableClient.execute( - entity.getAzureTableName().toString(), TableOperation.insert(entity)); - return tableResult.getHttpStatusCode() == Response.Status.CREATED.getStatusCode(); + return tableRefrence(entity.getAzureTableName()) + .execute(TableOperation.insert(entity)) + .getHttpStatusCode() == Response.Status.NO_CONTENT.getStatusCode(); } catch (StorageException e) { LOG.warn("Error performing operation on Storage service", e); } @@ -50,8 +58,8 @@ public boolean insert(TableType entity) { public boolean insertOrReplace(TableType entity) { try { - final TableResult tableResult = cloudTableClient.execute( - entity.getAzureTableName().toString(), TableOperation.insertOrReplace(entity)); + final TableResult tableResult = tableRefrence(entity.getAzureTableName()) + .execute(TableOperation.insertOrReplace(entity)); switch (Response.Status.fromStatusCode(tableResult.getHttpStatusCode())) { case CREATED: case NO_CONTENT: @@ -67,12 +75,9 @@ public boolean insertOrReplace(TableType entity) { public Optional retrieve(TableKey tableKey) { try { - final TableResult tableResult = cloudTableClient.execute( - tableKey.getTable().toString(), - TableOperation.retrieve( - tableKey.getPartitionKey(), tableKey.getRowKey(), tableKey.getEntityClass())); - final EntityType entityType = tableResult.getResultAsType(); - return Optional.fromNullable(entityType); + return Optional.fromNullable(tableRefrence(tableKey.getTable()) + .execute(TableOperation.retrieve(tableKey.getPartitionKey(), tableKey.getRowKey(), tableKey.getEntityClass())) + .getResultAsType()); } catch (StorageException e) { LOG.warn("Error retrieving entity from table: {}", tableKey.getTable(), e); } @@ -81,10 +86,9 @@ public Optional retrieve(Tab public boolean update(TableType entity) { try { - final TableResult result = cloudTableClient.execute( - entity.getAzureTableName().toString(), - TableOperation.replace(entity)); - return result.getHttpStatusCode() == Response.Status.NO_CONTENT.getStatusCode(); + return tableRefrence(entity.getAzureTableName()) + .execute(TableOperation.replace(entity)) + .getHttpStatusCode() == Response.Status.NO_CONTENT.getStatusCode(); } catch (StorageException e) { LOG.warn("Error updating row in table: {}", entity.getAzureTableName(), e); } @@ -93,10 +97,7 @@ public boolean update(TableType entity) { public boolean destroy(AzureTableName azureTableName) { try { - final CloudTable cloudTable = new CloudTable(azureTableName.toString(), cloudTableClient); - return cloudTable.deleteIfExists(); - } catch (URISyntaxException e) { - LOG.warn("Invalid Azure table URL specified {}", azureTableName, e); + return tableRefrence(azureTableName).deleteIfExists(); } catch (StorageException e) { LOG.warn("Error deleting table from Azure", e); } @@ -105,18 +106,16 @@ public boolean destroy(AzureTableName azureTableName) { public boolean exists(AzureTableName azureTableName) { try { - final CloudTable cloudTable = new CloudTable(azureTableName.toString(), cloudTableClient); - return cloudTable.exists(); - } catch (URISyntaxException e) { - LOG.warn("Invalid Azure table URL specified {}", azureTableName, e); + return tableRefrence(azureTableName).exists(); } catch (StorageException e) { LOG.warn("Error accessing azure table", e); } throw new IllegalStateException("Error verifying if table " + azureTableName + " exists"); } - public ImmutableList search(TableQuery query) { - return ImmutableList.copyOf(cloudTableClient.execute(query)); + public ImmutableList search(AzureTableName azureTableName, + TableQuery query) { + return ImmutableList.copyOf(tableRefrence(azureTableName).execute(query)); } public Iterable listTables() { @@ -125,9 +124,9 @@ public Iterable listTables() { public boolean remove(TableType entity) { try { - final TableResult result = cloudTableClient.execute( - entity.getAzureTableName().toString(), TableOperation.delete(entity)); - return result.getHttpStatusCode() == Response.Status.NO_CONTENT.getStatusCode(); + return tableRefrence(entity.getAzureTableName()) + .execute(TableOperation.delete(entity)) + .getHttpStatusCode() == Response.Status.NO_CONTENT.getStatusCode(); } catch (StorageException e) { LOG.warn("Error updating row in table: {}", entity.getAzureTableName(), e); } diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClientFactory.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClientFactory.java index 0ba8516..6ba1030 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClientFactory.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/TableClientFactory.java @@ -1,8 +1,10 @@ package com.yammer.breakerbox.azure; -import com.microsoft.windowsazure.services.core.storage.CloudStorageAccount; -import com.microsoft.windowsazure.services.core.storage.RetryLinearRetry; -import com.microsoft.windowsazure.services.table.client.CloudTableClient; +import com.google.common.primitives.Ints; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.RetryLinearRetry; +import com.microsoft.azure.storage.table.CloudTableClient; +import com.microsoft.azure.storage.table.TableRequestOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,11 +25,12 @@ public TableClient create() { final CloudStorageAccount storageAccount = new CloudStorageAccount(azureTableConfiguration.getStorageCredentialsAccountAndKey(), true); final CloudTableClient cloudTableClient = storageAccount.createCloudTableClient(); - cloudTableClient.setRetryPolicyFactory( - new RetryLinearRetry( - (int) azureTableConfiguration.getRetryInterval().toMilliseconds(), - azureTableConfiguration.getRetryAttempts())); - cloudTableClient.setTimeoutInMs((int) azureTableConfiguration.getTimeout().toMilliseconds()); + final TableRequestOptions defaultOptions = new TableRequestOptions(); + defaultOptions.setRetryPolicyFactory(new RetryLinearRetry( + Ints.checkedCast(azureTableConfiguration.getRetryInterval().toMilliseconds()), + azureTableConfiguration.getRetryAttempts())); + defaultOptions.setTimeoutIntervalInMs(Ints.checkedCast(azureTableConfiguration.getTimeout().toMilliseconds())); + cloudTableClient.setDefaultRequestOptions(defaultOptions); return new TableClient(cloudTableClient); } catch (URISyntaxException err) { LOGGER.error("Failed to create a TableClient", err); diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableKey.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableKey.java index 59f4045..5f36c52 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableKey.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableKey.java @@ -1,6 +1,6 @@ package com.yammer.breakerbox.azure.core; -import com.microsoft.windowsazure.services.table.client.TableServiceEntity; +import com.microsoft.azure.storage.table.TableServiceEntity; public interface TableKey { String getRowKey(); diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableType.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableType.java index 68c2fb6..8d71c1d 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableType.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/core/TableType.java @@ -1,6 +1,6 @@ package com.yammer.breakerbox.azure.core; -import com.microsoft.windowsazure.services.table.client.TableServiceEntity; +import com.microsoft.azure.storage.table.TableServiceEntity; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/DependencyEntity.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/DependencyEntity.java index e638b2b..2887486 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/DependencyEntity.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/DependencyEntity.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; -import com.microsoft.windowsazure.services.table.client.TableServiceEntity; +import com.microsoft.azure.storage.table.TableServiceEntity; import com.yammer.breakerbox.azure.core.AzureTableName; import com.yammer.breakerbox.azure.core.TableId; import com.yammer.breakerbox.azure.core.TableKey; diff --git a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/ServiceEntity.java b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/ServiceEntity.java index a5e07fc..2960884 100644 --- a/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/ServiceEntity.java +++ b/breakerbox-azure/src/main/java/com/yammer/breakerbox/azure/model/ServiceEntity.java @@ -1,6 +1,6 @@ package com.yammer.breakerbox.azure.model; -import com.microsoft.windowsazure.services.table.client.TableServiceEntity; +import com.microsoft.azure.storage.table.TableServiceEntity; import com.yammer.breakerbox.azure.core.AzureTableName; import com.yammer.breakerbox.azure.core.TableId; import com.yammer.breakerbox.azure.core.TableKey; diff --git a/breakerbox-azure/src/test/java/com/yammer/breakerbox/azure/model/tests/DependencyEntityTest.java b/breakerbox-azure/src/test/java/com/yammer/breakerbox/azure/model/tests/DependencyEntityTest.java index 5027f6a..e71c64c 100644 --- a/breakerbox-azure/src/test/java/com/yammer/breakerbox/azure/model/tests/DependencyEntityTest.java +++ b/breakerbox-azure/src/test/java/com/yammer/breakerbox/azure/model/tests/DependencyEntityTest.java @@ -1,7 +1,7 @@ package com.yammer.breakerbox.azure.model.tests; import com.google.common.base.Optional; -import com.microsoft.windowsazure.services.table.client.TableServiceEntity; +import com.microsoft.azure.storage.table.TableServiceEntity; import com.yammer.breakerbox.azure.TableClient; import com.yammer.breakerbox.azure.TableClientFactory; import com.yammer.breakerbox.azure.model.DependencyEntity; diff --git a/pom.xml b/pom.xml index 79b7f14..5331a62 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ 1.1.3 - 0.4.6 + 5.4.0 1.1.3 1.0.0 1.0.5 @@ -117,23 +117,9 @@ ${h2.version} - com.microsoft.windowsazure - microsoft-windowsazure-api - ${windowsazure.version} - - - stax - stax-api - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-json - - + com.microsoft.azure + azure-storage + ${azure-storage.version} io.fabric8 From c11b3ef42420253decd0d79ec3cfd2eee06e8fd7 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Fri, 4 Aug 2017 12:01:44 -0700 Subject: [PATCH 2/2] update changelog with azure-storage --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 4a07898..6f04de3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ Next --- * [Removed hardcoding of rancher query parameters](https://github.com/yammer/breakerbox/pull/42) Thanks @priyadarsh * [Dropwizard 1.1.3](https://github.com/dropwizard/dropwizard/releases/tag/v1.1.3) +* [azure-storage-java 5.4.0](https://github.com/Azure/azure-storage-java/releases/tag/v5.4.0) 0.6.3 -- July 13, 2017 ---