use base api client

package.json

@@ -49,8 +49,7 @@
     "@types/to-json-schema": "^0.2.4"
   },
   "dependencies": {
-    "@lifeomic/attempt": "^3.0.3",
-    "node-fetch": "2",
+    "@jupiterone/integration-sdk-http-client": "^13.2.0",
     "to-json-schema": "^0.2.5"
   }
 }

src/client.ts

@@ -1,113 +1,57 @@
-import { URL, URLSearchParams } from 'url';
-import fetch, { Response } from 'node-fetch';
-import { retry } from '@lifeomic/attempt';
-
 import {
+  IntegrationLogger,
   IntegrationProviderAPIError,
   IntegrationProviderAuthenticationError,
 } from '@jupiterone/integration-sdk-core';
 
 import { IntegrationConfig } from './config';
 import { AppInstance, Device, NetskopeResponse, UserConfig } from './types';
+import { BaseAPIClient } from '@jupiterone/integration-sdk-http-client';
 
 export type ResourceIteratee<T> = (each: T) => Promise<void> | void;
 
-class ResponseError extends IntegrationProviderAPIError {
-  response: Response;
-  constructor(options) {
-    super(options);
-    this.response = options.response;
-  }
-}
+const ENTITIES_PER_PAGE = 500;
 
-export class APIClient {
-  constructor(readonly config: IntegrationConfig) {}
+export class APIClient extends BaseAPIClient {
+  private readonly token: string;
 
-  private readonly paginateEntitiesPerPage = 500;
+  constructor(config: IntegrationConfig, logger: IntegrationLogger) {
+    super({
+      baseUrl: `https://${config.tenantName}.goskope.com/api/v1`,
+      logger,
+      logErrorBody: true,
+    });
+    this.token = config.apiV1Token;
+  }
 
-  private withBaseUri = (path: string, params?: Record<string, string>) => {
-    const url = new URL(
-      `https://${this.config.tenantName}.goskope.com/api/v1${path}`,
-    );
-    url.search = new URLSearchParams(params).toString();
-    return url.toString();
-  };
+  protected getAuthorizationHeaders(): Record<string, string> {
+    // Not fully implemented because netskope API doesn't authenticate through headers
+    return {};
+  }
 
-  public async request<T>(uri: string): Promise<T> {
-    try {
-      const result = await retry<Response>(
-        async () => {
-          // Only POST requests can have body, where token is placed
-          const authBody = { token: this.config.apiV1Token };
-          const response = await fetch(uri, {
-            method: 'POST',
-            body: JSON.stringify(authBody),
-            headers: { 'Content-Type': 'application/json' },
-          });
-          if (!response.ok) {
-            throw new ResponseError({
-              endpoint: uri,
-              status: response.status,
-              statusText: response.statusText,
-              response,
-            });
-          }
-          return response;
-        },
-        {
-          delay: 1000,
-          maxAttempts: 10,
-        },
-      );
-      return (await result.json()) as T;
-    } catch (err) {
-      throw new IntegrationProviderAPIError({
-        cause: err,
-        endpoint: uri.toString(),
-        status: err.status,
-        statusText: err.statusText,
-      });
-    }
+  public async postRequest<T>(uri: string): Promise<T> {
+    const response = await this.retryableRequest(uri, {
+      method: 'POST',
+      body: {
+        token: this.token,
+      },
+      authorize: false, // don't set Authorization headers
+    });
+    return response.json();
   }
 
   public async getRequest<T>(uri: string): Promise<T> {
-    try {
-      const result = await retry<Response>(
-        async () => {
-          const response = await fetch(uri, {
-            method: 'GET',
-            headers: { 'Content-Type': 'application/json' },
-          });
-          if (!response.ok) {
-            throw new ResponseError({
-              endpoint: uri,
-              status: response.status,
-              statusText: response.statusText,
-              response,
-            });
-          }
-          return response;
-        },
-        {
-          delay: 1000,
-          maxAttempts: 10,
-        },
-      );
-      return (await result.json()) as T;
-    } catch (err) {
-      throw new IntegrationProviderAPIError({
-        cause: err,
-        endpoint: uri.toString(),
-        status: err.status,
-        statusText: err.statusText,
-      });
-    }
+    const response = await this.retryableRequest(uri, {
+      method: 'GET',
+      authorize: false, // don't set Authorization headers
+    });
+    return response.json();
   }
 
   public async verifyAuthentication(): Promise<void> {
-    const endpoint = this.withBaseUri('/clients', { limit: '1' });
+    const endpoint = '/clients?limit=1';
     try {
-      const body = await this.request<NetskopeResponse<Device[]>>(endpoint);
+      const body = await this.postRequest<NetskopeResponse<Device[]>>(endpoint);
       if (body.status === 'error') {
         throw new IntegrationProviderAuthenticationError({
           endpoint,
@@ -130,12 +74,10 @@ export class APIClient {
     let length = 0;
 
     do {
-      const endpoint = this.withBaseUri('/clients', {
-        limit: `${this.paginateEntitiesPerPage}`,
-        skip: `${this.paginateEntitiesPerPage * page}`,
-      });
+      const skip = ENTITIES_PER_PAGE * page;
+      const endpoint = `/clients?limit=${ENTITIES_PER_PAGE}&skip=${skip}`;
 
-      const body = await this.request<NetskopeResponse<Device[]>>(endpoint);
+      const body = await this.postRequest<NetskopeResponse<Device[]>>(endpoint);
 
       if (body.status === 'error') {
         throw new IntegrationProviderAPIError({
@@ -158,12 +100,8 @@ export class APIClient {
     email: string,
     iteratee: ResourceIteratee<UserConfig>,
   ) {
-    const endpoint = this.withBaseUri('/userconfig', {
-      email,
-      configtype: 'agent',
-    });
-
-    const body = await this.request<NetskopeResponse<UserConfig>>(endpoint);
+    const endpoint = `/userconfig?email=${email}&configtype=agent`;
+    const body = await this.postRequest<NetskopeResponse<UserConfig>>(endpoint);
 
     if (body.status === 'success') {
       await iteratee(body.data);
@@ -175,14 +113,11 @@ export class APIClient {
     let length = 0;
 
     do {
-      const endpoint = this.withBaseUri('/app_instances', {
-        op: 'list',
-        limit: `${this.paginateEntitiesPerPage}`,
-        skip: `${this.paginateEntitiesPerPage * page}`,
-      });
+      const skip = ENTITIES_PER_PAGE * page;
+      const endpoint = `/app_instances?op=list&limit=${ENTITIES_PER_PAGE}&skip=${skip}`;
 
       const body =
-        await this.request<NetskopeResponse<AppInstance[]>>(endpoint);
+        await this.postRequest<NetskopeResponse<AppInstance[]>>(endpoint);
 
       if (body.status === 'error') {
         throw new IntegrationProviderAPIError({
@@ -207,12 +142,8 @@ export class APIClient {
     let length = 0;
 
     do {
-      const endpoint = this.withBaseUri('/app_instances', {
-        op: 'list',
-        limit: `${this.paginateEntitiesPerPage}`,
-        skip: `${this.paginateEntitiesPerPage * page}`,
-        token: this.config.apiV1Token,
-      });
+      const skip = ENTITIES_PER_PAGE * page;
+      const endpoint = `/app_instances?op=list&limit=${ENTITIES_PER_PAGE}&skip=${skip}&token=${this.token}`;
 
       const body =
         await this.getRequest<NetskopeResponse<AppInstance[]>>(endpoint);
@@ -235,6 +166,13 @@ export class APIClient {
   }
 }
 
-export function createAPIClient(config: IntegrationConfig): APIClient {
-  return new APIClient(config);
+let client: APIClient | undefined;
+export function createAPIClient(
+  config: IntegrationConfig,
+  logger: IntegrationLogger,
+): APIClient {
+  if (!client) {
+    client = new APIClient(config, logger);
+  }
+  return client;
 }

src/config.ts

@@ -35,6 +35,6 @@ export async function validateInvocation(
     );
   }
 
-  const apiClient = createAPIClient(config);
+  const apiClient = createAPIClient(config, context.logger);
   await apiClient.verifyAuthentication();
 }

src/steps/app-instance/index.ts

@@ -19,8 +19,9 @@ import { createAppInstanceEntity } from './converter';
 export async function fetchAppInstances({
   instance,
   jobState,
+  logger,
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
-  const apiClient = createAPIClient(instance.config);
+  const apiClient = createAPIClient(instance.config, logger);
   const tenantEntity = (await jobState.getData(TENANT_ENTITY_KEY)) as Entity;
 
   await apiClient.iterateGetAppInstances(async (appInstance) => {

src/steps/device/index.ts

@@ -19,8 +19,9 @@ import { createDeviceEntity } from './converter';
 export async function fetchDevices({
   instance,
   jobState,
+  logger,
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
-  const apiClient = createAPIClient(instance.config);
+  const apiClient = createAPIClient(instance.config, logger);
   const tenantEntity = (await jobState.getData(TENANT_ENTITY_KEY)) as Entity;
 
   await apiClient.iterateDevices(async (device) => {

src/steps/user-configuration/index.test.ts

@@ -1,3 +1,4 @@
+jest.setTimeout(60_000);
 import { executeStepWithDependencies } from '@jupiterone/integration-sdk-testing';
 import { buildStepTestConfigForStep } from '../../../test/config';
 import { Recording, setupProjectRecording } from '../../../test/recording';

src/steps/user-configuration/index.ts

@@ -17,7 +17,7 @@ export async function fetchUserConfiguration({
   jobState,
   logger,
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
-  const apiClient = createAPIClient(instance.config);
+  const apiClient = createAPIClient(instance.config, logger);
 
   await jobState.iterateEntities(
     { _type: Entities.USER._type },

yarn.lock

@@ -1390,6 +1390,11 @@
     ajv "^8.0.0"
     ajv-formats "^2.0.0"
 
+"@jupiterone/hierarchical-token-bucket@^0.3.1":
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/@jupiterone/hierarchical-token-bucket/-/hierarchical-token-bucket-0.3.1.tgz#b4bda4b5b1eed2fde3db776ce7241162c83b9b01"
+  integrity sha512-iD8Dqaggb3yuhx457OUNyDUspnLBhkT6dgFzwHzqFLGwa4a5CCev44+sYVOQ9SbAwOR+Yv5URJH3IXHKh1yxPg==
+
 "@jupiterone/integration-sdk-cli@^13.2.0":
   version "13.2.0"
   resolved "https://registry.yarnpkg.com/@jupiterone/integration-sdk-cli/-/integration-sdk-cli-13.2.0.tgz#8ab1c12dcc5ad9766c67581e6a7d08607d7148a3"
@@ -1456,6 +1461,18 @@
     ajv-formats "^3.0.1"
     prettier "^3.2.5"
 
+"@jupiterone/integration-sdk-http-client@^13.2.0":
+  version "13.2.0"
+  resolved "https://registry.yarnpkg.com/@jupiterone/integration-sdk-http-client/-/integration-sdk-http-client-13.2.0.tgz#1873ba2e73a069f92f4ff161798dcadaa7dbbbd7"
+  integrity sha512-9wd+yLww3H2FsswcuphDFuGIaPe5UB5joaincRz6r41k6HcId3tjavRJEleKJGZWfXxpHl4aFbxZU5+30SwV8Q==
+  dependencies:
+    "@jupiterone/hierarchical-token-bucket" "^0.3.1"
+    "@jupiterone/integration-sdk-core" "^13.2.0"
+    "@lifeomic/attempt" "^3.0.3"
+    form-data "^4.0.0"
+    lodash "^4.17.21"
+    node-fetch "^2.7.0"
+
 "@jupiterone/integration-sdk-runtime@^13.2.0":
   version "13.2.0"
   resolved "https://registry.yarnpkg.com/@jupiterone/integration-sdk-runtime/-/integration-sdk-runtime-13.2.0.tgz#ec96a48968a666b53fdc5db54b71b842f4c8d7f3"
@@ -5651,10 +5668,10 @@ nock@^13.2.1:
     lodash "^4.17.21"
     propagate "^2.0.0"
 
-node-fetch@2:
-  version "2.6.7"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
-  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
+node-fetch@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
   dependencies:
     whatwg-url "^5.0.0"