From 7a00df04cdfb221293cf4c93e0fea1853f68deb4 Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Tue, 16 Apr 2024 15:47:16 -0500 Subject: [PATCH] feature: manage OCI instance pools --- src/cloud_instance_manager_selector.ts | 12 ++ src/oracle_instance_pool_manager.ts | 182 +++++++++++++++++++++++ src/test/oracle_instance_pool_manager.ts | 51 +++++++ 3 files changed, 245 insertions(+) create mode 100644 src/oracle_instance_pool_manager.ts create mode 100644 src/test/oracle_instance_pool_manager.ts diff --git a/src/cloud_instance_manager_selector.ts b/src/cloud_instance_manager_selector.ts index c9b7c73..086ffa0 100644 --- a/src/cloud_instance_manager_selector.ts +++ b/src/cloud_instance_manager_selector.ts @@ -3,6 +3,7 @@ import CustomInstanceManager from './custom_instance_manager'; import NomadInstanceManager from './nomad_instance_manager'; import DigitalOceanInstanceManager from './digital_ocean_instance_manager'; import { CloudInstanceManager } from './cloud_instance_manager'; +import OracleInstancePoolManager from './oracle_instance_pool_manager'; export interface CloudInstanceManagerSelectorOptions { cloudProviders: string[]; @@ -19,6 +20,7 @@ export interface CloudInstanceManagerSelectorOptions { export class CloudInstanceManagerSelector { private oracleInstanceManager: OracleInstanceManager; + private oracleInstancePoolManager: OracleInstancePoolManager; private digitalOceanInstanceManager: DigitalOceanInstanceManager; private customInstanceManager: CustomInstanceManager; private nomadInstanceManager: NomadInstanceManager; @@ -32,6 +34,14 @@ export class CloudInstanceManagerSelector { }); } + if (options.cloudProviders.includes('oraclepool')) { + this.oracleInstancePoolManager = new OracleInstancePoolManager({ + isDryRun: options.isDryRun, + ociConfigurationFilePath: options.ociConfigurationFilePath, + ociConfigurationProfile: options.ociConfigurationProfile, + }); + } + if (options.cloudProviders.includes('custom')) { this.customInstanceManager = new CustomInstanceManager({ isDryRun: options.isDryRun, @@ -57,6 +67,8 @@ export class CloudInstanceManagerSelector { switch (cloud) { case 'oracle': return this.oracleInstanceManager; + case 'oraclepool': + return this.oracleInstancePoolManager; case 'digitalocean': return this.digitalOceanInstanceManager; case 'nomad': diff --git a/src/oracle_instance_pool_manager.ts b/src/oracle_instance_pool_manager.ts new file mode 100644 index 0000000..fe70dcd --- /dev/null +++ b/src/oracle_instance_pool_manager.ts @@ -0,0 +1,182 @@ +import core = require('oci-core'); +import common = require('oci-common'); +import { InstanceGroup } from './instance_group'; +import { Context } from './context'; +import { CloudRetryStrategy } from './cloud_manager'; +import { CloudInstanceManager, CloudInstance } from './cloud_instance_manager'; +import { workrequests } from 'oci-sdk'; + +const maxTimeInSeconds = 60 * 60; // The duration for waiter configuration before failing. Currently set to 1 hour. +const maxDelayInSeconds = 30; // The max delay for the waiter configuration. Currently set to 30 seconds + +const waiterConfiguration: common.WaiterConfiguration = { + terminationStrategy: new common.MaxTimeTerminationStrategy(maxTimeInSeconds), + delayStrategy: new common.ExponentialBackoffDelayStrategy(maxDelayInSeconds), +}; + +export interface OracleInstancePoolManagerOptions { + isDryRun: boolean; + ociConfigurationFilePath: string; + ociConfigurationProfile: string; +} + +export default class OracleInstancePoolManager implements CloudInstanceManager { + private isDryRun: boolean; + private provider: common.ConfigFileAuthenticationDetailsProvider; + private computeManagementClient: core.ComputeManagementClient; + private workRequestClient: workrequests.WorkRequestClient; + + constructor(options: OracleInstancePoolManagerOptions) { + this.isDryRun = options.isDryRun; + this.provider = new common.ConfigFileAuthenticationDetailsProvider( + options.ociConfigurationFilePath, + options.ociConfigurationProfile, + ); + this.computeManagementClient = new core.ComputeManagementClient({ + authenticationDetailsProvider: this.provider, + }); + this.workRequestClient = new workrequests.WorkRequestClient({ + authenticationDetailsProvider: this.provider, + }); + + this.launchInstances = this.launchInstances.bind(this); + } + + async detachInstance(ctx: Context, group: InstanceGroup, instance: string): Promise { + ctx.logger.info(`[oraclepool] Detaching instance ${instance}`); + this.computeManagementClient.regionId = group.region; + + const cwaiter = this.computeManagementClient.createWaiters(this.workRequestClient, waiterConfiguration); + const response = await cwaiter.forDetachInstancePoolInstance({ + instancePoolId: group.instanceConfigurationId, + detachInstancePoolInstanceDetails: { instanceId: instance }, + }); + ctx.logger.info(`[oraclepool] Finished detaching instance ${instance}`, { response }); + } + + async launchInstances( + ctx: Context, + group: InstanceGroup, + groupCurrentCount: number, + quantity: number, + ): Promise> { + ctx.logger.info(`[oraclepool] Launching a batch of ${quantity} instances in group ${group.name}`); + + this.computeManagementClient.regionId = group.region; + const poolDetails = await this.computeManagementClient.getInstancePool({ + instancePoolId: group.instanceConfigurationId, + }); + + const poolInstances = await this.computeManagementClient.listInstancePoolInstances({ + compartmentId: group.compartmentId, + instancePoolId: group.instanceConfigurationId, + }); + + const existingInstanceIds = poolInstances.items.map((instance) => { + return instance.id; + }); + + ctx.logger.debug(`[oraclepool] Instance pool ${group.name} instances`, { instances: poolInstances.items }); + + const newSize = quantity + groupCurrentCount; + if (groupCurrentCount == poolDetails.instancePool.size) { + ctx.logger.debug(`[oraclepool] Instance pool ${group.name} size matches current count`, { + current: groupCurrentCount, + size: poolDetails.instancePool.size, + newSize, + }); + } else { + ctx.logger.error(`[oraclepool] Instance pool ${group.name} size DOES NOT matches current count`, { + current: groupCurrentCount, + size: poolDetails.instancePool.size, + newSize, + }); + } + const updateResult = await this.computeManagementClient.updateInstancePool({ + instancePoolId: group.instanceConfigurationId, + updateInstancePoolDetails: { + size: newSize, + }, + }); + + ctx.logger.info(`[oraclepool] Updated instance pool size for group ${group.name}`, { updateResult }); + + this.workRequestClient.regionId = group.region; + const cwaiter = this.computeManagementClient.createWaiters(this.workRequestClient, waiterConfiguration); + const runningPool = await cwaiter.forInstancePool( + { + instancePoolId: group.instanceConfigurationId, + }, + core.models.InstancePool.LifecycleState.Running, + ); + + ctx.logger.info(`[oraclepool] Instance pool for ${group.name} back in running state`, { runningPool }); + + if (runningPool.instancePool.size == newSize) { + ctx.logger.debug(`[oraclepool] Instance pool ${group.name} size matches new size`, { + newSize, + }); + } else { + ctx.logger.error(`[oraclepool] Instance pool ${group.name} size DOES NOT matches new size`, { + newSize, + }); + } + + const newPoolInstances = await this.computeManagementClient.listInstancePoolInstances({ + compartmentId: group.compartmentId, + instancePoolId: group.instanceConfigurationId, + }); + + const result = newPoolInstances.items + .map((instance) => { + return instance.id; + }) + .filter((instanceId) => { + return !existingInstanceIds.includes(instanceId); + }); + + ctx.logger.info(`[oraclepool] Finished launching all the instances in group ${group.name}`, { result }); + + return result; + } + + async getInstances( + ctx: Context, + group: InstanceGroup, + cloudRetryStrategy: CloudRetryStrategy, + ): Promise> { + const computeManagementClient = new core.ComputeManagementClient( + { + authenticationDetailsProvider: this.provider, + }, + { + retryConfiguration: { + terminationStrategy: new common.MaxTimeTerminationStrategy(cloudRetryStrategy.maxTimeInSeconds), + delayStrategy: new common.ExponentialBackoffDelayStrategy(cloudRetryStrategy.maxDelayInSeconds), + retryCondition: (response) => { + return ( + cloudRetryStrategy.retryableStatusCodes.filter((retryableStatusCode) => { + return response.statusCode === retryableStatusCode; + }).length > 0 + ); + }, + }, + }, + ); + computeManagementClient.regionId = group.region; + + const poolInstances = await computeManagementClient.listInstancePoolInstances({ + compartmentId: group.compartmentId, + instancePoolId: group.instanceConfigurationId, + }); + + return poolInstances.items.map((instance) => { + ctx.logger.debug('Found instance in oracle pool', { instance }); + return { + instanceId: instance.id, + displayName: instance.displayName, + cloudStatus: instance.state, + }; + }); + } +} diff --git a/src/test/oracle_instance_pool_manager.ts b/src/test/oracle_instance_pool_manager.ts new file mode 100644 index 0000000..94fd129 --- /dev/null +++ b/src/test/oracle_instance_pool_manager.ts @@ -0,0 +1,51 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + +import assert from 'node:assert'; +import test, { afterEach, describe, mock } from 'node:test'; + +import OracleInstancePoolManager from '../oracle_instance_pool_manager'; + +function log(level, message, data) { + console.log(`${Date.now()} ${level}: ${message}`); + console.log(data); +} + +describe('InstancePoolManager', () => { + const manager = new OracleInstancePoolManager({ + isDryRun: true, + ociConfigurationFilePath: process.env.OCI_CONFIGURATION_FILE_PATH, + ociConfigurationProfile: process.env.OCI_CONFIGURATION_PROFILE, + }); + const context = { + logger: { + debug: mock.fn(log.bind('debug')), + info: mock.fn(log.bind('info')), + error: mock.fn(log.bind('error')), + }, + }; + + afterEach(() => { + mock.restoreAll(); + }); + + describe('getInstances', () => { + // This is a test for the getInstances method + test('will call the correct endpoint', async () => { + console.log('Starting getInstances test'); + const instances = await manager.getInstances( + context, + { + name: 'group', + region: process.env.REGION, + compartmentId: process.env.COMPARTMENT_OCID, + instanceConfigurationId: process.env.INSTANCE_POOL_ID, + }, + { maxAttempts: 1, maxTimeInSeconds: 60, maxDelayInSeconds: 30, retryableStatusCodes: [404, 429] }, + ); + console.log('ended getInstances test'); + assert.ok(instances); + console.log(instances); + }); + }); +});