Skip to content

Latest commit

 

History

History
401 lines (318 loc) · 15.3 KB

README.md

File metadata and controls

401 lines (318 loc) · 15.3 KB
Raw

Storage

Latest version Build Status Downloads per month

OST Storage contains DB storage libraries and respective services. It also contains data sharding libraries and services.

Constructor parameters:

There is 1 parameter required while creating the storage implementer.

  • First parameter is mandatory and it specifies the configuration strategy to be used. An example of the configStrategy is:
configStrategy = 
{
  "cache": {
    "engine": "memcached",
    "servers": [
      "127.0.0.1:11211"
    ],
    "defaultTtl": 36000
  },
  "storage": {
    "endpoint": "http://localhost:8000",
    "region": "localhost",
    "apiVersion": "2012-08-10",
    "apiKey": "X",
    "apiSecret": "X",
    "enableSsl": "0",
    "enableLogging": "0",
    "enableAutoscaling": "0",
    "tablePrefix":"X",
    "maxRetryCount":"1",
    "autoScaling": {
      "endpoint": "http://localhost:8000",
      "region": "localhost",
      "apiKey": "X",
      "apiSecret": "X",
      "apiVersion": "2012-08-10",
      "enableSsl": "0"
    }
  }
}

DynamoDB Apis

For all DynamoDB methods parameters description please refer AWS DynamoDB Docs

DynamoDB constructor

  params dynamodbConnectionParams

  const OSTStorage = require('@ostdotcom/storage');
  let storage = OSTStorage.getInstance( configStrategy );
  let ddbServiceObj = storage.dynamoDBService;

Create table

  params createTableParams

    //Create DynamoDB Table
    ddbServiceObj.createTable(createTableParams);

Create table migration

  Table migration needs autoScaleObject
  params createTableMigrationParams   params.createTableConfig
  params.updateContinuousBackupConfig
  params.autoScalingConfig.registerScalableTargetWrite
  params.autoScalingConfig.registerScalableTargetRead
  params.autoScalingConfig.putScalingPolicyWrite
  params.autoScalingConfig.putScalingPolicyRead
  params.autoScalingConfig.globalSecondaryIndex[<GSI_INDEX_NAME>].autoScalingConfig - Auto Scaling config of Global Secondary Indexes same as of auto scale table config


 // Create Table Migration
 // 1. Creates table
 // 2. Enables read/write auto scaling
 // 3. Returns describe table response 
 ddbServiceObj.createTableMigration(createTableMigrationParams);

Update table

  params updateTableParams

// Update DynamoDB Table
ddbServiceObj.updateTable(updateTableParams);

Describe table

  params describeTableParams

// Describe DynamoDB Table 
ddbServiceObj.describeTable(describeTableParams);

List tables

  params listTableParams

// List DynamoDB Tables
ddbServiceObj.listTables(listTableParams);

Update Continuous Backups

  params updateContinuousParams

// Point in time recovery for DynamoDB Table
ddbServiceObj.updateContinuousBackups(updateContinuousParams);

Delete table

  params deleteTableParams

// Delete DynamoDB table
ddbServiceObj.deleteTable(deleteTableParams);

Batch Get Item

  params batchGetItemParams

// Batch Get Item
ddbServiceObj.batchGetItem(batchGetItemParams);

Batch Write Item

  params batchWriteItemParams
  params unprocessedItemsRetryCount Retry count for unprocessed Items

// Batch Write Item
ddbServiceObj.batchWriteItem(batchWriteItemParams, unprocessedRetryCount);

Query

  params queryParams

// Query Items
ddbServiceObj.query(queryParams);

Scan

  params scanParams

// Scan Items
ddbServiceObj.scan(scanParams);

Put Item

  params putItemParams

// Put Items
ddbServiceObj.putItem(putItemParams);

Update Item

  params updateItemParams

// Update an Item
ddbServiceObj.updateItem(updateItemParams);

Delete Item

  params deleteItemParams

// Delete Items
ddbServiceObj.deleteItem(deleteItemParams);

Table Exists Using WaitFor

  params tableExistsParams

// Check if table exists and is in ACTIVE state using wait for method
ddbServiceObj.tableExistsUsingWaitFor(tableExistsParams);

Table Does not Exists Using WaitFor

  params tableNotExistsParams

// Check if table doesn't exists using wait for method
ddbServiceObj.tableNotExistsUsingWaitFor(tableNotExistsParams);

Check Table Exists

  params tableExistParams

// Check if table exist in ACTIVE state using describe table method
// If table is being created, then response will be false
ddbServiceObj.checkTableExist(tableExistParams);

Shard Management Apis

Shard Management AvailableShards Table Schema

{
        TableName: "available_shards",
          AttributeDefinitions: [
        {
          AttributeName: "shardName",
          AttributeType: "S"
        },
        {
          AttributeName: "entityType",
          AttributeType: "S"
        },
        {
          AttributeName: "allocationType",
          AttributeType: "N"
        }
      ],
        KeySchema: [
        {
          AttributeName: "shardName",
          KeyType: "HASH"
        }
      ],
        GlobalSecondaryIndexes: [{
        IndexName: "available_shard_entity_type_allocation_type_index",
        KeySchema: [
          {
            AttributeName: "entityType",
            KeyType: 'HASH'
          },
          {
            AttributeName: "allocationType",
            KeyType: 'RANGE'
          }
        ],
        Projection: {
          ProjectionType: 'KEYS_ONLY'
        },
        ProvisionedThroughput: {
          ReadCapacityUnits: 1,
          WriteCapacityUnits: 1
        }
      }],
        ProvisionedThroughput: {
        ReadCapacityUnits: 1,
          WriteCapacityUnits: 1
      }
    }

Shard Management Managed shard table schema

{
      TableName: "managed_shards",
      AttributeDefinitions: [
        {
          AttributeName: "identifier",
          AttributeType: "S"
        },
        {
          AttributeName: "entityType",
          AttributeType: "S"
        }
      ],
      KeySchema: [
        {
          AttributeName: "identifer",
          KeyType: "HASH"
        },
        {
          AttributeName: "entityType",
          KeyType: "RANGE"
        }
      ],
      ProvisionedThroughput: {
        ReadCapacityUnits: 1,
        WriteCapacityUnits: 1
      }
    }

DynamoDB And AutoScaling constructor

  DynamoDB params dynamodbConnectionParams
  AutoScaling params autoScalingConnectionParams

  const OSTStorage = require('@ostdotcom/storage');
  let storage = OSTStorage.getInstance( configStrategy );
  let ddbServiceObj = storage.dynamoDBService;
  let shardManagementObj = ddbServiceObj.shardManagement();
  let autoScalingObj = storage.autoScalingService;

Run shard migration

  Shard migration params ddbServiceObj and autoScalingObj

// Run Shard Migration
// Created available_shards and managed_shards table
shardManagementObj.runShardMigration(ddbServiceObj, autoScalingObj);

Add shard

  addShardParams as JSON params
  params.shard_name(String) - Shard name to be added
  params.entity_type(String) - Entity type to be assigned to shard

// Add Shard
// Creates item in available_shards table
shardManagementObj.addShard(addShardParams);

Configure shard

  configureShardParams as JSON params
  params.shard_name(String) - Shard name to be added
  params.allocation_type(Enum) - Allocation type of Shard :- if
   enabled: Provided shard is available for multiple assignment,
   disabled: Provided shard is dedicated shard for single identifier

// Configure Shard
// Configure Enable/Disable allocation type
shardManagementObj.configureShard(configureShardParams);

Assign shard

  assignShardParams as JSON params
  params.identifier(String) - Identifier to be assigned to shard   params.shard_name(String) - Shard name to be assigned
  params.entity_type(String) - Entity type of the shard
  params.force_assignment(Boolean) - (Optional default: false) Pass true if shard is dedicated and assignment needs to be done.
   Note: It should be used in case dedicated shard is assigned first time.

// Assign Shard to an identifier
// Creates entry in managed_shards table
shardManagementObj.assignShard(assignShardParams);

Get Shards By Type

  getShardsByTypeParams as JSON params
  params.entity_type(String) - Entity type to be assigned to shard
  params.shard_type(Enum) - Shard type :- if
   all: give all available shards,
   enabled: Shard is available for multiple assignment,
   disabled: Shard is dedicated for single Id

// Get Shards By Different Types
// Type Values : all/enabled/disabled
shardManagementObj.getShardsByType(getShardsByTypeParams);

Has shard

  hasShardParams as JSON params
  params.shard_names(Array{String}) - List of shard names to be queried for existence.

// Does this shard exist in available_shards table
shardManagementObj.hasShard(hasShardParams);

Get Managed Shard

  managedShardParams as JSON params
  params.entity_type(String) - Entity type of the shard to be queried
  params.identifiers(Array) - List of Identifiers to be queried 

// Get Managed shards
shardManagementObj.getManagedShard(managedShardParams);

Steps for Adding and Configuring New Shard

  1. Call runShardMigration if shard migrations are not done already. ApiRef
    This will create available_shards and managed_shards table in DynamoDB.
  2. Create Shard Table. ApiRef
  3. Call addShard api. This will add a item in available_shards table. Shard will be added in disabled state. ApiRef
  4. Call configureShard with allocation_type='enabled' if it's a shared shard. For dedicated shard keep allocation_type='disabled'. ApiRef
  5. Call assignShard to assign shard to a client/identifier. This creates an item in managed_shards table. ApiRef
  6. Call getManagedShard to get shardName for a client/identifier. ApiRef

Auto Scaling Apis

For Parameters description please refer AWS DynamoDB Docs

AutoScaling constructor

  params autoScalingConnectionParams

  const OSTStorage = require('@ostdotcom/storage');
  let storage = OSTStorage.getInstance( configStrategy );
  let autoScalingObj = storage.autoScalingService;

Register Scalable Target

  params registerScalableTargetParams

// Registers or updates a scalable target. Scalable target is a resource that Application Auto Scaling can scale out or scale in. After you have registered a scalable target, you can use this operation to update the minimum and maximum values for its scalable dimension.
autoScalingObj.registerScalableTarget(registerScalableTargetParams);

Put Scaling Policy

  params putScalingPolicyParams

// Creates or updates a policy for an Application Auto Scaling scalable target
autoScalingObj.putScalingPolicy(putScalingPolicyParams);

Deregister Scalable Target

  params deregisterScalableTargetParams

// Deregistering a scalable target deletes the scaling policies that are associated with it.
autoScalingObj.deregisterScalableTarget(deregisterScalableTargetParams);

Delete Scaling Policy

  params deleteScalingPolicyParams

// Deletes the specified Application Auto Scaling scaling policy
autoScalingObj.deleteScalingPolicy(deleteScalingPolicyParams);

Describe Scalable Targets

  params describeScalableTargetsParams

// Gets information about the scalable targets in the specified namespace. 
autoScalingObj.describeScalableTargets(describeScalableTargetsParams);

Describe Scaling Policies

  params describeScalingPoliciesParams

// Describes the scaling policies for the specified service namespace.
autoScalingObj.describeScalingPolicies(describeScalingPoliciesParams);