From af005993cd43e9c5f341af5c5ebbc446fafd4fa6 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Thu, 8 Feb 2024 08:20:14 +0100 Subject: [PATCH] refactor proxy service so proxying doesn't require access to db (#1602) * refactor proxy service so proxying doesn't require access to db We don't want runners to have direct access to the datbase. The goal of this commit is to remove database dependency when calling the proxy service, while still executing the http request directly without going through our own API * string equality without type conversion Co-authored-by: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> * string equality without type conversion Co-authored-by: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> * fix: await log function * proxy.service: remove unecessary if(activityLogId) * refactor proxy.service to remove db access from response handlers * refactor linear post connection * remove redundant attributes --------- Co-authored-by: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> --- .../lib/controllers/proxy.controller.ts | 282 +++----- .../scripts/connection/connection.manager.ts | 16 +- .../github-app-oauth-post-connection.ts | 10 +- .../connection/hubspot-post-connection.ts | 10 +- .../connection/jira-post-connection.ts | 14 +- .../connection/linear-post-connection.ts | 8 +- .../github-app-oauth-webhook-routing.ts | 2 +- packages/shared/lib/models/Proxy.ts | 17 +- packages/shared/lib/sdk/sync.ts | 61 +- packages/shared/lib/sdk/sync.unit.test.ts | 5 +- .../shared/lib/services/config.service.ts | 19 +- packages/shared/lib/services/proxy.service.ts | 601 ++++++------------ .../lib/services/proxy.service.unit.test.ts | 193 +++++- packages/shared/lib/utils/error.ts | 2 +- 14 files changed, 579 insertions(+), 661 deletions(-) diff --git a/packages/server/lib/controllers/proxy.controller.ts b/packages/server/lib/controllers/proxy.controller.ts index 3afbb6f4ab..a452badb73 100644 --- a/packages/server/lib/controllers/proxy.controller.ts +++ b/packages/server/lib/controllers/proxy.controller.ts @@ -3,12 +3,15 @@ import type { OutgoingHttpHeaders } from 'http'; import stream, { Readable, Transform, TransformCallback, PassThrough } from 'stream'; import url, { UrlWithParsedQuery } from 'url'; import querystring from 'querystring'; -import axios, { AxiosError, AxiosResponse } from 'axios'; +import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'; import { backOff } from 'exponential-backoff'; +import { ActivityLogMessage, NangoError } from '@nangohq/shared'; +import { updateProvider as updateProviderActivityLog, updateEndpoint as updateEndpointActivityLog } from '@nangohq/shared'; import { createActivityLog, createActivityLogMessageAndEnd, + createActivityLogMessage, updateSuccess as updateSuccessActivityLog, HTTP_VERB, LogLevel, @@ -21,8 +24,9 @@ import { InternalProxyConfiguration, ApplicationConstructedProxyConfiguration, ErrorSourceEnum, - ServiceResponse, - proxyService + proxyService, + connectionService, + configService } from '@nangohq/shared'; interface ForwardedHeaders { @@ -45,8 +49,8 @@ class ProxyController { const retries = req.get('Retries') as string; const baseUrlOverride = req.get('Base-Url-Override') as string; const decompress = req.get('Decompress') as string; - const isSync = req.get('Nango-Is-Sync') as string; - const isDryRun = req.get('Nango-Is-Dry-Run') as string; + const isSync = (req.get('Nango-Is-Sync') as string) === 'true'; + const isDryRun = (req.get('Nango-Is-Dry-Run') as string) === 'true'; const existingActivityLogId = req.get('Nango-Activity-Log-Id') as number | string; const environment_id = getEnvironmentId(res); const accountId = getAccount(res); @@ -93,34 +97,61 @@ class ProxyController { method: method.toUpperCase() as HTTP_VERB }; + const { + success: connSuccess, + error: connError, + response: connection + } = await connectionService.getConnectionCredentials(accountId, environment_id, connectionId, providerConfigKey, activityLogId, logAction, false); + + if (!connSuccess || !connection) { + throw new Error(`Failed to get connection credentials: '${connError}'`); + } + const providerConfig = await configService.getProviderConfig(providerConfigKey, environment_id); + + if (!providerConfig) { + await createActivityLogMessageAndEnd({ + level: 'error', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: 'Provider configuration not found' + }); + throw new NangoError('unknown_provider_config'); + } + await updateProviderActivityLog(activityLogId as number, providerConfig.provider); + const internalConfig: InternalProxyConfiguration = { existingActivityLogId: activityLogId as number, - environmentId: environment_id, - accountId, - throwErrors: false, - isFlow: isSync === 'true', - isDryRun: isDryRun === 'true' + connection, + provider: providerConfig.provider }; - const { - success, - error, - response: configBody - } = (await proxyService.routeOrConfigure(externalConfig, internalConfig)) as ServiceResponse; - - if (!success || !configBody) { + const { success, error, response: proxyConfig, activityLogs } = proxyService.configure(externalConfig, internalConfig); + if (activityLogId) { + await updateEndpointActivityLog(activityLogId, externalConfig.endpoint); + for (const log of activityLogs) { + switch (log.level) { + case 'error': + await createActivityLogMessageAndEnd(log); + break; + default: + await createActivityLogMessage(log); + break; + } + } + } + if (!success || !proxyConfig || error) { errorManager.errResFromNangoErr(res, error); - return; } - await this.sendToHttpMethod(res, next, method as HTTP_VERB, configBody, activityLogId as number, environment_id, isSync, isDryRun); + await this.sendToHttpMethod(res, next, method as HTTP_VERB, proxyConfig, activityLogId as number, environment_id, isSync, isDryRun); } catch (error) { const environmentId = getEnvironmentId(res); const connectionId = req.get('Connection-Id') as string; const providerConfigKey = req.get('Provider-Config-Key') as string; - await errorManager.report(error, { + errorManager.report(error, { source: ErrorSourceEnum.PLATFORM, operation: LogActionEnum.PROXY, environmentId, @@ -149,8 +180,8 @@ class ProxyController { configBody: ApplicationConstructedProxyConfiguration, activityLogId: number, environment_id: number, - isSync?: string, - isDryRun?: string + isSync?: boolean, + isDryRun?: boolean ) { const url = proxyService.constructUrl(configBody); let decompress = false; @@ -159,17 +190,7 @@ class ProxyController { decompress = true; } - if (method === 'POST') { - return this.post(res, next, url, configBody, activityLogId, environment_id, decompress, isSync, isDryRun); - } else if (method === 'PATCH') { - return this.patch(res, next, url, configBody, activityLogId, environment_id, decompress, isSync, isDryRun); - } else if (method === 'PUT') { - return this.put(res, next, url, configBody, activityLogId, environment_id, decompress, isSync, isDryRun); - } else if (method === 'DELETE') { - return this.delete(res, next, url, configBody, activityLogId, environment_id, decompress, isSync, isDryRun); - } else { - return this.get(res, next, url, configBody, activityLogId, environment_id, decompress, isSync, isDryRun); - } + return this.request(res, next, method, url, configBody, activityLogId, environment_id, decompress, isSync, isDryRun, configBody.data); } private async handleResponse( @@ -179,8 +200,8 @@ class ProxyController { activityLogId: number, environment_id: number, url: string, - isSync?: string, - isDryRun?: string + isSync?: boolean, + isDryRun?: boolean ) { if (!isSync) { await updateSuccessActivityLog(activityLogId, true); @@ -275,155 +296,46 @@ class ProxyController { * Get * @param {Response} res Express response object * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. + * @param {HTTP_VERB} method * @param {string} url * @param {ApplicationConstructedProxyConfiguration} config */ - private async get( - res: Response, - _next: NextFunction, - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - decompress: boolean, - isSync?: string, - isDryRun?: string - ) { - try { - const headers = proxyService.constructHeaders(config); - - const responseStream: AxiosResponse = await backOff( - () => { - return axios({ - method: 'get', - url, - responseType: 'stream', - headers, - decompress - }); - }, - { numOfAttempts: Number(config.retries), retry: proxyService.retry.bind(this, activityLogId, environment_id, config) } - ); - - this.handleResponse(res, responseStream, config, activityLogId, environment_id, url, isSync, isDryRun); - } catch (e: unknown) { - this.handleErrorResponse(res, e, url, config, activityLogId, environment_id); - } - } - - /** - * Post - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async post( - res: Response, - _next: NextFunction, - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - decompress: boolean, - isSync?: string, - isDryRun?: string - ) { - try { - const headers = proxyService.constructHeaders(config); - const responseStream: AxiosResponse = await backOff( - () => { - return axios({ - method: 'post', - url, - data: config.data ?? {}, - responseType: 'stream', - headers, - decompress - }); - }, - { numOfAttempts: Number(config.retries), retry: proxyService.retry.bind(this, activityLogId, environment_id, config) } - ); - - this.handleResponse(res, responseStream, config, activityLogId, environment_id, url, isSync, isDryRun); - } catch (error) { - this.handleErrorResponse(res, error, url, config, activityLogId, environment_id); - } - } - - /** - * Patch - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async patch( - res: Response, - _next: NextFunction, - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - decompress: boolean, - isSync?: string, - isDryRun?: string - ) { - try { - const headers = proxyService.constructHeaders(config); - const responseStream: AxiosResponse = await backOff( - () => { - return axios({ - method: 'patch', - url, - data: config.data ?? {}, - responseType: 'stream', - headers, - decompress - }); - }, - { numOfAttempts: Number(config.retries), retry: proxyService.retry.bind(this, activityLogId, environment_id, config) } - ); - - this.handleResponse(res, responseStream, config, activityLogId, environment_id, url, isSync, isDryRun); - } catch (error) { - this.handleErrorResponse(res, error, url, config, activityLogId, environment_id); - } - } - /** - * Put - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async put( + private async request( res: Response, _next: NextFunction, + method: HTTP_VERB, url: string, config: ApplicationConstructedProxyConfiguration, activityLogId: number, environment_id: number, decompress: boolean, - isSync?: string, - isDryRun?: string + isSync?: boolean, + isDryRun?: boolean, + data?: unknown ) { try { + const activityLogs: ActivityLogMessage[] = []; const headers = proxyService.constructHeaders(config); + const requestConfig: AxiosRequestConfig = { + method, + url, + responseType: 'stream', + headers, + decompress + }; + if (['POST', 'PUT', 'PATCH'].includes(method)) { + requestConfig.data = data || {}; + } const responseStream: AxiosResponse = await backOff( () => { - return axios({ - method: 'put', - url, - data: config.data ?? {}, - responseType: 'stream', - headers, - decompress - }); + return axios(requestConfig); }, - { numOfAttempts: Number(config.retries), retry: proxyService.retry.bind(this, activityLogId, environment_id, config) } + { numOfAttempts: Number(config.retries), retry: proxyService.retry.bind(this, activityLogId, environment_id, config, activityLogs) } ); + activityLogs.forEach((activityLogMessage) => { + createActivityLogMessage(activityLogMessage); + }); this.handleResponse(res, responseStream, config, activityLogId, environment_id, url, isSync, isDryRun); } catch (error) { @@ -431,44 +343,6 @@ class ProxyController { } } - /** - * Delete - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async delete( - res: Response, - _next: NextFunction, - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - decompress: boolean, - isSync?: string, - isDryRun?: string - ) { - try { - const headers = proxyService.constructHeaders(config); - const responseStream: AxiosResponse = await backOff( - () => { - return axios({ - method: 'delete', - url, - responseType: 'stream', - headers, - decompress - }); - }, - { numOfAttempts: Number(config.retries), retry: proxyService.retry.bind(this, activityLogId, environment_id, config) } - ); - this.handleResponse(res, responseStream, config, activityLogId, environment_id, url, isSync, isDryRun); - } catch (e) { - this.handleErrorResponse(res, e, url, config, activityLogId, environment_id); - } - } - private async reportError( error: AxiosError, url: string, diff --git a/packages/shared/lib/integrations/scripts/connection/connection.manager.ts b/packages/shared/lib/integrations/scripts/connection/connection.manager.ts index 07bd3fa6bf..965153b863 100644 --- a/packages/shared/lib/integrations/scripts/connection/connection.manager.ts +++ b/packages/shared/lib/integrations/scripts/connection/connection.manager.ts @@ -1,4 +1,4 @@ -import type { AxiosResponse } from 'axios'; +import type { AxiosError, AxiosResponse } from 'axios'; import type { RecentlyCreatedConnection, Connection, ConnectionConfig } from '../../../models/Connection.js'; import { LogLevel, LogActionEnum } from '../../../models/Activity.js'; import { createActivityLogAndLogMessage } from '../../../services/activity/activity.service.js'; @@ -21,7 +21,7 @@ const handlers: PostConnectionHandlersMap = postConnectionHandlers as unknown as export interface InternalNango { getConnection: () => Promise; - proxy: ({ method, endpoint, data }: UserProvidedProxyConfiguration) => Promise; + proxy: ({ method, endpoint, data }: UserProvidedProxyConfiguration) => Promise; updateConnectionConfig: (config: ConnectionConfig) => Promise; } @@ -41,11 +41,8 @@ async function execute(createdConnection: RecentlyCreatedConnection, provider: s } const internalConfig = { - environmentId: createdConnection.environment_id, - isFlow: true, - isDryRun: false, - throwErrors: false, - connection + connection, + provider }; const externalConfig = { @@ -62,12 +59,13 @@ async function execute(createdConnection: RecentlyCreatedConnection, provider: s return connection as Connection; }, - proxy: ({ method, endpoint, data }: UserProvidedProxyConfiguration) => { + proxy: async ({ method, endpoint, data }: UserProvidedProxyConfiguration) => { const finalExternalConfig = { ...externalConfig, method: method || externalConfig.method, endpoint }; if (data) { finalExternalConfig.data = data; } - return proxyService.routeOrConfigure(finalExternalConfig, internalConfig) as Promise; + const { response } = await proxyService.route(finalExternalConfig, internalConfig); + return response; }, updateConnectionConfig: (connectionConfig: ConnectionConfig) => { return connectionService.updateConnectionConfig(connection as unknown as Connection, connectionConfig); diff --git a/packages/shared/lib/integrations/scripts/connection/github-app-oauth-post-connection.ts b/packages/shared/lib/integrations/scripts/connection/github-app-oauth-post-connection.ts index 86fc6fcf19..ffcee8e97a 100644 --- a/packages/shared/lib/integrations/scripts/connection/github-app-oauth-post-connection.ts +++ b/packages/shared/lib/integrations/scripts/connection/github-app-oauth-post-connection.ts @@ -1,19 +1,21 @@ import type { InternalNango as Nango } from './connection.manager.js'; import { AuthModes, OAuth2Credentials } from '../../../models/Auth.js'; +import axios from 'axios'; export default async function execute(nango: Nango) { + const connection = await nango.getConnection(); const response = await nango.proxy({ - endpoint: `/user` + endpoint: `/user`, + connectionId: connection.connection_id, + providerConfigKey: connection.provider_config_key }); - if (!response || !response.data) { + if (axios.isAxiosError(response) || !response || !response.data) { return; } const handle = response.data.login; - const connection = await nango.getConnection(); - let updatedConfig: Record = { handle }; diff --git a/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts b/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts index 23d6bac5d8..d31185fa4a 100644 --- a/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts +++ b/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts @@ -1,9 +1,15 @@ import type { InternalNango as Nango } from './connection.manager.js'; +import axios from 'axios'; export default async function execute(nango: Nango) { - const response = await nango.proxy({ endpoint: '/account-info/v3/details' }); + const connection = await nango.getConnection(); + const response = await nango.proxy({ + endpoint: '/account-info/v3/details', + connectionId: connection.connection_id, + providerConfigKey: connection.provider_config_key + }); - if (!response || !response.data || !response.data.portalId) { + if (axios.isAxiosError(response) || !response || !response.data || !response.data.portalId) { return; } const portalId = response.data.portalId; diff --git a/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts b/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts index 15ac64d170..1bec67ea33 100644 --- a/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts +++ b/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts @@ -1,21 +1,27 @@ import type { InternalNango as Nango } from './connection.manager.js'; +import axios from 'axios'; export default async function execute(nango: Nango) { + const connection = await nango.getConnection(); const response = await nango.proxy({ - endpoint: `oauth/token/accessible-resources` + endpoint: `oauth/token/accessible-resources`, + connectionId: connection.connection_id, + providerConfigKey: connection.provider_config_key }); - if (!response || !response.data || response.data.length === 0 || !response.data[0].id) { + if (axios.isAxiosError(response) || !response || !response.data || response.data.length === 0 || !response.data[0].id) { return; } const cloudId = response.data[0].id; const accountResponse = await nango.proxy({ - endpoint: `ex/jira/${cloudId}/rest/api/3/myself` + endpoint: `ex/jira/${cloudId}/rest/api/3/myself`, + connectionId: connection.connection_id, + providerConfigKey: connection.provider_config_key }); - if (!accountResponse || !accountResponse.data || accountResponse.data.length === 0) { + if (axios.isAxiosError(accountResponse) || !accountResponse || !accountResponse.data || accountResponse.data.length === 0) { await nango.updateConnectionConfig({ cloudId }); return; } diff --git a/packages/shared/lib/integrations/scripts/connection/linear-post-connection.ts b/packages/shared/lib/integrations/scripts/connection/linear-post-connection.ts index a08eabcc4e..7085e163fc 100644 --- a/packages/shared/lib/integrations/scripts/connection/linear-post-connection.ts +++ b/packages/shared/lib/integrations/scripts/connection/linear-post-connection.ts @@ -1,4 +1,5 @@ import type { InternalNango as Nango } from './connection.manager.js'; +import axios from 'axios'; export default async function execute(nango: Nango) { const query = ` @@ -8,13 +9,16 @@ export default async function execute(nango: Nango) { } }`; + const connection = await nango.getConnection(); const response = await nango.proxy({ endpoint: '/graphql', data: { query }, - method: 'POST' + method: 'POST', + connectionId: connection.connection_id, + providerConfigKey: connection.provider_config_key }); - if (!response || !response.data || !response.data.data?.organization?.id) { + if (axios.isAxiosError(response) || !response || !response.data || !response.data.data?.organization?.id) { return; } diff --git a/packages/shared/lib/integrations/scripts/webhook/github-app-oauth-webhook-routing.ts b/packages/shared/lib/integrations/scripts/webhook/github-app-oauth-webhook-routing.ts index 6ef5ac9dff..739d21e708 100644 --- a/packages/shared/lib/integrations/scripts/webhook/github-app-oauth-webhook-routing.ts +++ b/packages/shared/lib/integrations/scripts/webhook/github-app-oauth-webhook-routing.ts @@ -62,7 +62,7 @@ async function handleCreateWebhook(integration: ProviderConfig, body: any) { return; } - const template = await configService.getTemplate(integration?.provider as string); + const template = configService.getTemplate(integration?.provider as string); const activityLogId = connection.connection_config['pendingLog']; delete connection.connection_config['pendingLog']; diff --git a/packages/shared/lib/models/Proxy.ts b/packages/shared/lib/models/Proxy.ts index 14c1651966..a0cf19a9b2 100644 --- a/packages/shared/lib/models/Proxy.ts +++ b/packages/shared/lib/models/Proxy.ts @@ -5,6 +5,8 @@ import type { Connection } from './Connection.js'; import type { Template as ProviderTemplate } from './Provider.js'; interface BaseProxyConfiguration { + providerConfigKey: string; + connectionId: string; endpoint: string; retries?: number; data?: unknown; @@ -17,20 +19,13 @@ interface BaseProxyConfiguration { } export interface UserProvidedProxyConfiguration extends BaseProxyConfiguration { - providerConfigKey?: string; - connectionId?: string; - retries?: number; decompress?: boolean | string; - method?: 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE' | 'get' | 'post' | 'patch' | 'put' | 'delete'; paginate?: Partial | Partial | Partial; } export interface ApplicationConstructedProxyConfiguration extends BaseProxyConfiguration { - providerConfigKey: string; - connectionId: string; decompress?: boolean; - method: HTTP_VERB; provider: string; token: string | BasicApiCredentials | ApiKeyCredentials | AppCredentials; @@ -41,13 +36,9 @@ export interface ApplicationConstructedProxyConfiguration extends BaseProxyConfi export type ResponseType = 'arraybuffer' | 'blob' | 'document' | 'json' | 'text' | 'stream'; export interface InternalProxyConfiguration { - environmentId: number; - accountId?: number; - isFlow?: boolean; - isDryRun?: boolean; + provider: string; + connection: Connection; existingActivityLogId?: number; - throwErrors?: boolean; - connection?: Connection; } export interface RetryHeaderConfig { diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index c0db8abfbf..994a33696e 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -5,6 +5,7 @@ import proxyService from '../services/proxy.service.js'; import axios from 'axios'; import { getPersistAPIUrl, safeStringify } from '../utils/utils.js'; import type { IntegrationWithCreds } from '@nangohq/node/lib/types.js'; +import type { UserProvidedProxyConfiguration } from '../models/Proxy.js'; /* * @@ -289,42 +290,57 @@ export class NangoAction { }); } - public async proxy(config: ProxyConfiguration): Promise> { - const internalConfig = { - environmentId: this.environmentId as number, - isFlow: true, - isDryRun: this.dryRun as boolean, - existingActivityLogId: this.activityLogId as number, - throwErrors: true - }; - - let connection = undefined; - + private proxyConfig(config: ProxyConfiguration): UserProvidedProxyConfiguration { if (!config.connectionId && this.connectionId) { config.connectionId = this.connectionId; } - if (!config.providerConfigKey && this.providerConfigKey) { config.providerConfigKey = this.providerConfigKey; } + if (!config.connectionId) { + throw new Error('Missing connection id'); + } + if (!config.providerConfigKey) { + throw new Error('Missing provider config key'); + } + return { + ...config, + providerConfigKey: config.providerConfigKey, + connectionId: config.connectionId + }; + } + public async proxy(config: ProxyConfiguration): Promise> { if (this.dryRun) { return this.nango.proxy(config); } else { - const { connectionId, providerConfigKey } = config; - connection = await this.nango.getConnection(providerConfigKey as string, connectionId as string); - + const proxyConfig = this.proxyConfig(config); + const connection = await this.nango.getConnection(proxyConfig.providerConfigKey, proxyConfig.connectionId); if (!connection) { throw new Error(`Connection not found using the provider config key ${this.providerConfigKey} and connection id ${this.connectionId}`); } + const { + config: { provider } + } = await this.nango.getIntegration(proxyConfig.providerConfigKey); + + const { response, activityLogs: activityLogs } = await proxyService.route(proxyConfig, { + existingActivityLogId: this.activityLogId as number, + connection: connection, + provider + }); - const responseOrError = await proxyService.routeOrConfigure(config, { ...internalConfig, connection: connection as Connection }); + if (activityLogs) { + for (const log of activityLogs) { + if (log.level === 'debug') continue; + await this.log(log.content, { level: log.level }); + } + } - if (responseOrError instanceof Error) { - throw responseOrError; + if (response instanceof Error) { + throw response; } - return responseOrError as unknown as Promise>; + return response; } } @@ -509,20 +525,21 @@ export class NangoAction { updatedBodyOrParams[limitParameterName] = paginationConfig['limit']; } + const proxyConfig = this.proxyConfig(config); switch (paginationConfig.type.toLowerCase()) { case PaginationType.CURSOR: return yield* paginateService.cursor( - config, + proxyConfig, paginationConfig as CursorPagination, updatedBodyOrParams, passPaginationParamsInBody, this.proxy.bind(this) ); case PaginationType.LINK: - return yield* paginateService.link(config, paginationConfig, updatedBodyOrParams, passPaginationParamsInBody, this.proxy.bind(this)); + return yield* paginateService.link(proxyConfig, paginationConfig, updatedBodyOrParams, passPaginationParamsInBody, this.proxy.bind(this)); case PaginationType.OFFSET: return yield* paginateService.offset( - config, + proxyConfig, paginationConfig as OffsetPagination, updatedBodyOrParams, passPaginationParamsInBody, diff --git a/packages/shared/lib/sdk/sync.unit.test.ts b/packages/shared/lib/sdk/sync.unit.test.ts index 9f8ba0f522..56dc646646 100644 --- a/packages/shared/lib/sdk/sync.unit.test.ts +++ b/packages/shared/lib/sdk/sync.unit.test.ts @@ -14,6 +14,7 @@ vi.mock('@nangohq/node', () => { describe('Pagination', () => { const providerConfigKey = 'github'; + const connectionId = 'connection-1'; const cursorPagination: CursorPagination = { type: 'cursor', @@ -45,6 +46,7 @@ describe('Pagination', () => { secretKey: 'encrypted', serverUrl: 'https://example.com', providerConfigKey, + connectionId, dryRun: true }; nangoAction = new NangoAction(config); @@ -124,7 +126,8 @@ describe('Pagination', () => { endpoint, params: { offset: '3', per_page: 3 }, paginate: paginationConfigOverride, - providerConfigKey: 'github' + providerConfigKey, + connectionId }); }); diff --git a/packages/shared/lib/services/config.service.ts b/packages/shared/lib/services/config.service.ts index 4f0f05d82b..c3cfce0c2c 100644 --- a/packages/shared/lib/services/config.service.ts +++ b/packages/shared/lib/services/config.service.ts @@ -21,10 +21,25 @@ class ConfigService { public DEMO_GITHUB_CONFIG_KEY = 'demo-github-integration'; private getTemplatesFromFile() { - const templatesPath = path.join(dirname(), '../../../providers.yaml'); + const templatesPath = () => { + // find the providers.yaml file + // recursively searching in parent directories + const findProvidersYaml = (dir: string): string => { + const providersYamlPath = path.join(dir, 'providers.yaml'); + if (fs.existsSync(providersYamlPath)) { + return providersYamlPath; + } + const parentDir = path.dirname(dir); + if (parentDir === dir) { + throw new NangoError('providers_yaml_not_found'); + } + return findProvidersYaml(parentDir); + }; + return findProvidersYaml(dirname()); + }; try { - const fileEntries = yaml.load(fs.readFileSync(templatesPath).toString()) as { [key: string]: ProviderTemplate | ProviderTemplateAlias }; + const fileEntries = yaml.load(fs.readFileSync(templatesPath()).toString()) as { [key: string]: ProviderTemplate | ProviderTemplateAlias }; if (fileEntries == null) { throw new NangoError('provider_template_loading_failed'); diff --git a/packages/shared/lib/services/proxy.service.ts b/packages/shared/lib/services/proxy.service.ts index 271dcd9bb8..320a48c8f1 100644 --- a/packages/shared/lib/services/proxy.service.ts +++ b/packages/shared/lib/services/proxy.service.ts @@ -1,80 +1,101 @@ import axios, { AxiosError, AxiosResponse, AxiosRequestConfig, ParamsSerializerOptions } from 'axios'; import { backOff } from 'exponential-backoff'; import FormData from 'form-data'; - -import type { Connection } from '../models/Connection.js'; import { ApiKeyCredentials, BasicApiCredentials, AuthModes, OAuth2Credentials } from '../models/Auth.js'; import type { HTTP_VERB, ServiceResponse } from '../models/Generic.js'; import type { ResponseType, ApplicationConstructedProxyConfiguration, UserProvidedProxyConfiguration, InternalProxyConfiguration } from '../models/Proxy.js'; -import { LogAction, LogActionEnum } from '../models/Activity.js'; - -import { - createActivityLogMessageAndEnd, - createActivityLogMessage, - updateProvider as updateProviderActivityLog, - updateEndpoint as updateEndpointActivityLog -} from './activity/activity.service.js'; -import environmentService from './environment.service.js'; + import configService from './config.service.js'; -import connectionService from './connection.service.js'; import { interpolateIfNeeded, connectionCopyWithParsedConnectionConfig, mapProxyBaseUrlInterpolationFormat } from '../utils/utils.js'; import { NangoError } from '../utils/error.js'; +import type { ActivityLogMessage } from '../models/Activity.js'; +import type { Template as ProviderTemplate } from '../models/Provider.js'; + +interface Activities { + activityLogs: ActivityLogMessage[]; +} + +interface RouteResponse { + response: AxiosResponse | AxiosError; +} +interface RetryHandlerResponse { + shouldRetry: boolean; +} class ProxyService { - public async routeOrConfigure( + public async route( externalConfig: ApplicationConstructedProxyConfiguration | UserProvidedProxyConfiguration, internalConfig: InternalProxyConfiguration - ): Promise | AxiosResponse | AxiosError> { - const { success: validationSuccess, error: validationError } = await this.validateAndLog(externalConfig, internalConfig); - - const { throwErrors } = internalConfig; - - if (!validationSuccess) { - if (throwErrors) { - throw validationError; - } else { - return { success: false, error: validationError, response: null }; - } + ): Promise { + const { success, error, response: proxyConfig, activityLogs: configureActivityLogs } = this.configure(externalConfig, internalConfig); + if (!success || error || !proxyConfig) { + throw new Error(`Proxy configuration is missing: ${error}`); } + return await this.sendToHttpMethod(proxyConfig, internalConfig).then((resp) => { + return { response: resp.response, activityLogs: [...configureActivityLogs, ...resp.activityLogs] }; + }); + } + public configure( + externalConfig: ApplicationConstructedProxyConfiguration | UserProvidedProxyConfiguration, + internalConfig: InternalProxyConfiguration + ): ServiceResponse & Activities { + const activityLogs: ActivityLogMessage[] = []; let data = externalConfig.data; const { endpoint: passedEndpoint, providerConfigKey, connectionId, method, retries, headers, baseUrlOverride } = externalConfig; - const { environmentId: environment_id, accountId: optionalAccountId, isFlow, existingActivityLogId: activityLogId, isDryRun } = internalConfig; - const accountId = optionalAccountId ?? ((await environmentService.getAccountIdFromEnvironment(environment_id)) as number); - const logAction: LogAction = isFlow ? LogActionEnum.SYNC : LogActionEnum.PROXY; - - let endpoint = passedEndpoint; - let connection: Connection | null = null; - - // if this is a proxy call coming from a flow then the connection lookup - // is done before coming here. Otherwise we need to do it here. - if (!internalConfig.connection) { - const { success, error, response } = await connectionService.getConnectionCredentials( - accountId as number, - environment_id as number, - connectionId as string, - providerConfigKey as string, - activityLogId as number, - logAction, - false - ); + const { connection, provider, existingActivityLogId: activityLogId } = internalConfig; - if (!success) { - if (throwErrors) { - throw error; - } else { - return { success: false, error, response: null }; - } + if (!passedEndpoint && !baseUrlOverride) { + if (activityLogId) { + activityLogs.push({ + level: 'error', + environment_id: connection.environment_id, + activity_log_id: activityLogId, + timestamp: Date.now(), + content: 'Proxy: a API URL endpoint is missing.' + }); + } + return { success: false, error: new NangoError('missing_endpoint'), response: null, activityLogs }; + } + if (!connectionId) { + if (activityLogId) { + activityLogs.push({ + level: 'error', + environment_id: connection.environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `The connection id value is missing. If you're making a HTTP request then it should be included in the header 'Connection-Id'. If you're using the SDK the connectionId property should be specified.` + }); + } + return { success: false, error: new NangoError('missing_connection_id'), response: null, activityLogs }; + } + if (!providerConfigKey) { + if (activityLogId) { + activityLogs.push({ + level: 'error', + environment_id: connection.environment_id, + activity_log_id: activityLogId, + timestamp: Date.now(), + content: `The provider config key value is missing. If you're making a HTTP request then it should be included in the header 'Provider-Config-Key'. If you're using the SDK the providerConfigKey property should be specified.` + }); } + return { success: false, error: new NangoError('missing_provider_config_key'), response: null, activityLogs }; + } - connection = response; - } else { - connection = internalConfig.connection; + if (activityLogId) { + activityLogs.push({ + level: 'debug', + environment_id: connection.environment_id, + activity_log_id: activityLogId, + timestamp: Date.now(), + content: `Connection id: '${connectionId}' and provider config key: '${providerConfigKey}' parsed and received successfully` + }); } - let token; + let endpoint = passedEndpoint; - switch (connection?.credentials?.type) { + let token; + switch (connection.credentials?.type) { case AuthModes.OAuth2: { const credentials = connection.credentials as OAuth2Credentials; @@ -83,103 +104,69 @@ class ProxyService { break; case AuthModes.OAuth1: { const error = new Error('OAuth1 is not supported yet in the proxy.'); - if (throwErrors) { - throw error; - } else { - const nangoError = new NangoError('pass_through_error', error); - return { success: false, error: nangoError, response: null }; - } + const nangoError = new NangoError('pass_through_error', error); + return { success: false, error: nangoError, response: null, activityLogs }; } case AuthModes.Basic: - token = connection?.credentials; + token = connection.credentials; break; case AuthModes.ApiKey: - token = connection?.credentials; + token = connection.credentials; break; case AuthModes.App: { - const credentials = connection?.credentials; + const credentials = connection.credentials; token = credentials?.access_token; } break; } - if (!isFlow) { - await createActivityLogMessage({ - level: 'debug', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: 'Proxy: token retrieved successfully' - }); - } - - const providerConfig = await configService.getProviderConfig(providerConfigKey as string, environment_id); - - if (!providerConfig) { - await createActivityLogMessageAndEnd({ - level: 'error', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: 'Provider configuration not found' - }); - - if (throwErrors) { - throw new Error('Provider configuration not found'); - } else { - return { success: false, error: new NangoError('unknown_provider_config'), response: null }; - } - } - - await updateProviderActivityLog(activityLogId as number, String(providerConfig?.provider)); + activityLogs.push({ + level: 'debug', + environment_id: connection.environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: 'Proxy: token retrieved successfully' + }); - const template = configService.getTemplate(String(providerConfig?.provider)); + let template: ProviderTemplate | undefined; + try { + template = configService.getTemplate(provider); + } catch (error) {} - if ((!template.proxy || !template.proxy.base_url) && !baseUrlOverride) { - await createActivityLogMessageAndEnd({ + if (!template || ((!template.proxy || !template.proxy.base_url) && !baseUrlOverride)) { + activityLogs.push({ level: 'error', - environment_id, + environment_id: connection.environment_id, activity_log_id: activityLogId as number, timestamp: Date.now(), - content: `${Date.now()} The proxy is not supported for this provider ${String( - providerConfig?.provider - )}. You can easily add support by following the instructions at https://docs.nango.dev/contribute/nango-auth. -You can also use the baseUrlOverride to get started right away. -See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` + content: `${Date.now()} The proxy is not supported for this provider ${provider}. You can easily add support by following the instructions at https://docs.nango.dev/contribute/nango-auth. + You can also use the baseUrlOverride to get started right away. + See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` }); - const error = new NangoError('missing_base_api_url'); - if (throwErrors) { - throw error; - } else { - return { success: false, error, response: null }; - } + return { success: false, error: new NangoError('missing_base_api_url'), response: null, activityLogs }; } - if (!isFlow) { - await createActivityLogMessage({ - level: 'debug', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `Proxy: API call configuration constructed successfully with the base api url set to ${baseUrlOverride || template.proxy.base_url}` - }); - } + activityLogs.push({ + level: 'debug', + environment_id: connection.environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `Proxy: API call configuration constructed successfully with the base api url set to ${baseUrlOverride || template.proxy.base_url}` + }); if (!baseUrlOverride && template.proxy.base_url && endpoint.includes(template.proxy.base_url)) { endpoint = endpoint.replace(template.proxy.base_url, ''); } - if (!isFlow) { - await createActivityLogMessage({ - level: 'debug', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `Endpoint set to ${endpoint} with retries set to ${retries}` - }); - } + activityLogs.push({ + level: 'debug', + environment_id: connection.environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `Endpoint set to ${endpoint} with retries set to ${retries}` + }); if (headers && headers['Content-Type'] === 'multipart/form-data') { const formData = new FormData(); @@ -192,107 +179,28 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` } const configBody: ApplicationConstructedProxyConfiguration = { - endpoint: endpoint as string, + endpoint: endpoint, method: method?.toUpperCase() as HTTP_VERB, template, token: token || '', - provider: String(providerConfig?.provider), - providerConfigKey: String(providerConfigKey), - connectionId: String(connectionId), + provider: provider, + providerConfigKey: providerConfigKey, + connectionId: connectionId, headers: headers as Record, data, - retries: retries ? Number(retries) : 0, + retries: retries || 0, baseUrlOverride: baseUrlOverride as string, // decompress is used only when the call is truly a proxy call // Coming from a flow it is not a proxy call since the worker // makes the request so we don't allow an override in that case decompress: (externalConfig as UserProvidedProxyConfiguration).decompress === 'true' || externalConfig.decompress === true, - connection: connection as Connection, + connection: connection, params: externalConfig.params as Record, paramsSerializer: externalConfig.paramsSerializer as ParamsSerializerOptions, responseType: externalConfig.responseType as ResponseType }; - if (isFlow && !isDryRun) { - return this.sendToHttpMethod(configBody, internalConfig); - } else { - return { success: true, error: null, response: configBody }; - } - } - - public async validateAndLog( - externalConfig: ApplicationConstructedProxyConfiguration | UserProvidedProxyConfiguration, - internalConfig: InternalProxyConfiguration - ): Promise> { - const { existingActivityLogId: activityLogId, environmentId: environment_id } = internalConfig; - if (!externalConfig.endpoint && !externalConfig.baseUrlOverride) { - await createActivityLogMessageAndEnd({ - level: 'error', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: 'Proxy: a API URL endpoint is missing.' - }); - - const error = new NangoError('missing_endpoint'); - - if (internalConfig.throwErrors) { - throw error; - } else { - return { success: false, error, response: null }; - } - } - await updateEndpointActivityLog(activityLogId as number, externalConfig.endpoint); - - if (!externalConfig.connectionId) { - await createActivityLogMessageAndEnd({ - level: 'error', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `The connection id value is missing. If you're making a HTTP request then it should be included in the header 'Connection-Id'. If you're using the SDK the connectionId property should be specified.` - }); - - const error = new NangoError('missing_connection_id'); - - if (internalConfig.throwErrors) { - throw error; - } else { - return { success: false, error, response: null }; - } - } - - if (!externalConfig.providerConfigKey) { - await createActivityLogMessageAndEnd({ - level: 'error', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `The provider config key value is missing. If you're making a HTTP request then it should be included in the header 'Provider-Config-Key'. If you're using the SDK the providerConfigKey property should be specified.` - }); - - const error = new NangoError('missing_provider_config_key'); - - if (internalConfig.throwErrors) { - throw error; - } else { - return { success: false, error, response: null }; - } - } - - const { connectionId, providerConfigKey } = externalConfig; - - if (!internalConfig.isFlow) { - await createActivityLogMessage({ - level: 'debug', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `Connection id: '${connectionId}' and provider config key: '${providerConfigKey}' parsed and received successfully` - }); - } - - return { success: true, error: null, response: null }; + return { success: true, error: null, response: configBody, activityLogs }; } public retryHandler = async ( @@ -301,7 +209,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` error: AxiosError, type: 'at' | 'after', retryHeader: string - ): Promise => { + ): Promise => { if (type === 'at') { const resetTimeEpoch = error?.response?.headers[retryHeader] || error?.response?.headers[retryHeader.toLowerCase()]; @@ -314,17 +222,19 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` const content = `Rate limit reset time was parsed successfully, retrying after ${waitDuration} seconds`; - await createActivityLogMessage({ - level: 'error', - environment_id, - activity_log_id: activityLogId, - timestamp: Date.now(), - content - }); + const activityLogs: ActivityLogMessage[] = [ + { + level: 'error', + environment_id, + activity_log_id: activityLogId, + timestamp: Date.now(), + content + } + ]; await new Promise((resolve) => setTimeout(resolve, waitDuration * 1000)); - return true; + return { shouldRetry: true, activityLogs: activityLogs }; } } } @@ -336,21 +246,23 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` const retryAfter = Number(retryHeaderVal); const content = `Retry header was parsed successfully, retrying after ${retryAfter} seconds`; - await createActivityLogMessage({ - level: 'error', - environment_id, - activity_log_id: activityLogId, - timestamp: Date.now(), - content - }); + const activityLogs: ActivityLogMessage[] = [ + { + level: 'error', + environment_id, + activity_log_id: activityLogId, + timestamp: Date.now(), + content + } + ]; await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000)); - return true; + return { shouldRetry: true, activityLogs: activityLogs }; } } - return true; + return { shouldRetry: true, activityLogs: [] }; }; /** @@ -364,6 +276,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` activityLogId: number, environment_id: number, config: ApplicationConstructedProxyConfiguration, + activityLogs: ActivityLogMessage[], error: AxiosError, attemptNumber: number ): Promise => { @@ -380,14 +293,30 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` const type = config.retryHeader.at ? 'at' : 'after'; const retryHeader = config.retryHeader.at ? config.retryHeader.at : config.retryHeader.after; - return this.retryHandler(activityLogId, environment_id, error, type, retryHeader as string); + const { shouldRetry, activityLogs: retryActivityLogs } = await this.retryHandler( + activityLogId, + environment_id, + error, + type, + retryHeader as string + ); + retryActivityLogs.forEach((a: ActivityLogMessage) => activityLogs.push(a)); + return shouldRetry; } if (config.template.proxy && config.template.proxy.retry && (config.template.proxy?.retry?.at || config.template.proxy?.retry?.after)) { const type = config.template.proxy.retry.at ? 'at' : 'after'; const retryHeader = config.template.proxy.retry.at ? config.template.proxy.retry.at : config.template.proxy.retry.after; - return this.retryHandler(activityLogId, environment_id, error, type, retryHeader as string); + const { shouldRetry, activityLogs: retryActivityLogs } = await this.retryHandler( + activityLogId, + environment_id, + error, + type, + retryHeader as string + ); + retryActivityLogs.forEach((a: ActivityLogMessage) => activityLogs.push(a)); + return shouldRetry; } const content = `API received an ${error?.response?.status || error?.code} error, ${ @@ -396,7 +325,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` : 'but no retries will occur because retries defaults to 0 or were set to 0' }`; - await createActivityLogMessage({ + activityLogs.push({ level: 'error', environment_id, activity_log_id: activityLogId, @@ -419,7 +348,10 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` * @param {HTTP_VERB} method * @param {ApplicationConstructedProxyConfiguration} configBody */ - private sendToHttpMethod(configBody: ApplicationConstructedProxyConfiguration, internalConfig: InternalProxyConfiguration) { + private sendToHttpMethod( + configBody: ApplicationConstructedProxyConfiguration, + internalConfig: InternalProxyConfiguration + ): Promise { const options: AxiosRequestConfig = { headers: configBody.headers as Record }; @@ -436,22 +368,20 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` options.responseType = configBody.responseType; } - const url = this.constructUrl(configBody); + if (configBody.data) { + options.data = configBody.data; + } - const { existingActivityLogId: activityLogId, environmentId: environment_id } = internalConfig; + const { existingActivityLogId: activityLogId, connection } = internalConfig; const { method } = configBody; - if (method === 'POST') { - return this.post(url, configBody, activityLogId as number, environment_id, options); - } else if (method === 'PATCH') { - return this.patch(url, configBody, activityLogId as number, environment_id, options); - } else if (method === 'PUT') { - return this.put(url, configBody, activityLogId as number, environment_id, options); - } else if (method === 'DELETE') { - return this.delete(url, configBody, activityLogId as number, environment_id, options); - } else { - return this.get(url, configBody, activityLogId as number, environment_id, options); - } + options.url = this.constructUrl(configBody); + options.method = method; + + const headers = this.constructHeaders(configBody); + options.headers = { ...options.headers, ...headers }; + + return this.request(configBody, activityLogId as number, connection.environment_id, options); } public stripSensitiveHeaders(headers: ApplicationConstructedProxyConfiguration['headers'], config: ApplicationConstructedProxyConfiguration) { @@ -478,149 +408,27 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` return safeHeaders; } - /** - * Get - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async get( - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - options: AxiosRequestConfig - ) { - try { - const headers = this.constructHeaders(config); - - const response: AxiosResponse = await backOff( - () => { - return axios.get(url, { ...options, headers }); - }, - { numOfAttempts: Number(config.retries), retry: this.retry.bind(this, activityLogId, environment_id, config) } - ); - - return this.handleResponse(response, config, activityLogId, environment_id, url); - } catch (e: unknown) { - return this.handleErrorResponse(e as AxiosError, url, config, activityLogId, environment_id); - } - } - - /** - * Post - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async post( - url: string, + private async request( config: ApplicationConstructedProxyConfiguration, activityLogId: number, environment_id: number, options: AxiosRequestConfig - ) { + ): Promise { + const activityLogs: ActivityLogMessage[] = []; try { - const headers = this.constructHeaders(config); const response: AxiosResponse = await backOff( () => { - return axios.post(url, config.data ?? {}, { ...options, headers }); + return axios.request(options); }, - { numOfAttempts: Number(config.retries), retry: this.retry.bind(this, activityLogId, environment_id, config) } + { numOfAttempts: Number(config.retries), retry: this.retry.bind(this, activityLogId, environment_id, config, activityLogs) } ); - - return this.handleResponse(response, config, activityLogId, environment_id, url); - } catch (e: unknown) { - return this.handleErrorResponse(e as AxiosError, url, config, activityLogId, environment_id); - } - } - - /** - * Patch - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async patch( - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - options: AxiosRequestConfig - ) { - try { - const headers = this.constructHeaders(config); - const response: AxiosResponse = await backOff( - () => { - return axios.patch(url, config.data ?? {}, { ...options, headers }); - }, - { numOfAttempts: Number(config.retries), retry: this.retry.bind(this, activityLogId, environment_id, config) } - ); - - return this.handleResponse(response, config, activityLogId, environment_id, url); - } catch (e: unknown) { - return this.handleErrorResponse(e as AxiosError, url, config, activityLogId, environment_id); - } - } - - /** - * Put - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {pplicationConstructedProxyConfiguration} config - */ - private async put( - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - options: AxiosRequestConfig - ) { - try { - const headers = this.constructHeaders(config); - const response: AxiosResponse = await backOff( - () => { - return axios.put(url, config.data ?? {}, { ...options, headers }); - }, - { numOfAttempts: Number(config.retries), retry: this.retry.bind(this, activityLogId, environment_id, config) } - ); - - return this.handleResponse(response, config, activityLogId, environment_id, url); - } catch (e: unknown) { - return this.handleErrorResponse(e as AxiosError, url, config, activityLogId, environment_id); - } - } - - /** - * Delete - * @param {Response} res Express response object - * @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline. - * @param {string} url - * @param {ApplicationConstructedProxyConfiguration} config - */ - private async delete( - url: string, - config: ApplicationConstructedProxyConfiguration, - activityLogId: number, - environment_id: number, - options: AxiosRequestConfig - ) { - try { - const headers = this.constructHeaders(config); - const response: AxiosResponse = await backOff( - () => { - return axios.delete(url, { ...options, headers }); - }, - { numOfAttempts: Number(config.retries), retry: this.retry.bind(this, activityLogId, environment_id, config) } - ); - - return this.handleResponse(response, config, activityLogId, environment_id, url); + return this.handleResponse(response, config, activityLogId, environment_id, options.url!).then((resp) => { + return { response: resp.response, activityLogs: [...activityLogs, ...resp.activityLogs] }; + }); } catch (e: unknown) { - return this.handleErrorResponse(e as AxiosError, url, config, activityLogId, environment_id); + return this.handleErrorResponse(e as AxiosError, options.url!, config, activityLogId, environment_id).then((resp) => { + return { response: resp.response, activityLogs: [...activityLogs, ...resp.activityLogs] }; + }); } } @@ -724,10 +532,10 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` activityLogId: number, environment_id: number, url: string - ): Promise { + ): Promise { const safeHeaders = this.stripSensitiveHeaders(config.headers, config); - await createActivityLogMessageAndEnd({ + const activityLog: ActivityLogMessage = { level: 'info', environment_id, activity_log_id: activityLogId, @@ -736,22 +544,26 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` params: { headers: JSON.stringify(safeHeaders) } - }); + }; - return response; + return { + response, + activityLogs: [activityLog] + }; } - private async reportError( + private reportError( error: AxiosError, url: string, config: ApplicationConstructedProxyConfiguration, activityLogId: number, environment_id: number, errorMessage: string - ) { + ): ActivityLogMessage[] { + const activities: ActivityLogMessage[] = []; if (activityLogId) { const safeHeaders = this.stripSensitiveHeaders(config.headers, config); - await createActivityLogMessageAndEnd({ + activities.push({ level: 'error', environment_id, activity_log_id: activityLogId, @@ -771,6 +583,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` }`; console.error(content); } + return activities; } private async handleErrorResponse( @@ -779,7 +592,8 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` config: ApplicationConstructedProxyConfiguration, activityLogId: number, environment_id: number - ): Promise { + ): Promise { + const activityLogs: ActivityLogMessage[] = []; if (!error?.response?.data) { const { message, @@ -792,7 +606,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` const errorObject = { message, stack, code, status, url, method }; if (activityLogId) { - await createActivityLogMessageAndEnd({ + activityLogs.push({ level: 'error', environment_id, activity_log_id: activityLogId, @@ -804,7 +618,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` console.error(`Error: ${method.toUpperCase()} request to ${url} failed with the following params: ${JSON.stringify(errorObject)}`); } - await this.reportError(error, url, config, activityLogId, environment_id, message); + activityLogs.push(...this.reportError(error, url, config, activityLogId, environment_id, message)); } else { const { message, @@ -813,7 +627,7 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` const errorData = error?.response?.data; if (activityLogId) { - await createActivityLogMessageAndEnd({ + activityLogs.push({ level: 'error', environment_id, activity_log_id: activityLogId, @@ -825,10 +639,13 @@ See https://docs.nango.dev/guides/proxy#proxy-requests for more information.` console.error(`Error: ${method.toUpperCase()} request to ${url} failed with the following params: ${JSON.stringify(errorData)}`); } - await this.reportError(error, url, config, activityLogId, environment_id, message); + activityLogs.push(...this.reportError(error, url, config, activityLogId, environment_id, message)); } - return error; + return { + response: error, + activityLogs + }; } } diff --git a/packages/shared/lib/services/proxy.service.unit.test.ts b/packages/shared/lib/services/proxy.service.unit.test.ts index 005e4ad579..0ca781fcc8 100644 --- a/packages/shared/lib/services/proxy.service.unit.test.ts +++ b/packages/shared/lib/services/proxy.service.unit.test.ts @@ -1,10 +1,10 @@ import { expect, describe, it } from 'vitest'; import proxyService from './proxy.service.js'; -import { HTTP_VERB, AuthModes } from '../models/index.js'; +import { HTTP_VERB, AuthModes, UserProvidedProxyConfiguration, InternalProxyConfiguration, OAuth2Credentials } from '../models/index.js'; import type { ApplicationConstructedProxyConfiguration } from '../models/Proxy.js'; import type { AxiosError, AxiosResponse, InternalAxiosRequestConfig } from 'axios'; -describe('Proxy Controller Construct Header Tests', () => { +describe('Proxy service Construct Header Tests', () => { it('Should correctly construct a header using an api key with multiple headers', () => { const config = { endpoint: 'https://api.nangostarter.com', @@ -210,7 +210,7 @@ describe('Proxy Controller Construct Header Tests', () => { }); }); -describe('Proxy Controller Construct URL Tests', () => { +describe('Proxy service Construct URL Tests', () => { it('should correctly construct url with no trailing slash and no leading slash', () => { const config = { template: { @@ -481,7 +481,7 @@ describe('Proxy Controller Construct URL Tests', () => { await proxyService.retryHandler(1, 1, mockAxiosError, 'after', 'x-rateLimit-reset-after'); const after = Date.now(); const diff = after - before; - expect(diff).toBeGreaterThan(1000); + expect(diff).toBeGreaterThanOrEqual(1000); expect(diff).toBeLessThan(2000); }); @@ -506,3 +506,188 @@ describe('Proxy Controller Construct URL Tests', () => { expect(diff).toBeLessThan(2000); }); }); + +describe('Proxy service configure', () => { + it('Should fail if no endpoint', () => { + const externalConfig: UserProvidedProxyConfiguration = { + method: 'GET', + providerConfigKey: 'provider-config-key-1', + connectionId: 'connection-1', + endpoint: '' + }; + const internalConfig: InternalProxyConfiguration = { + provider: 'provider-1', + connection: { + environment_id: 1, + connection_id: 'connection-1', + provider_config_key: 'provider-config-key-1', + credentials: {} as OAuth2Credentials, + connection_config: {} + }, + existingActivityLogId: 1 + }; + const { success, error, response, activityLogs } = proxyService.configure(externalConfig, internalConfig); + expect(success).toBe(false); + expect(response).toBeNull(); + expect(error).toBeDefined(); + expect(error?.message).toContain('missing_endpoint'); + expect(activityLogs.length).toBe(1); + expect(activityLogs[0]).toMatchObject({ + environment_id: 1, + activity_log_id: 1, + level: 'error' + }); + }); + it('Should fail if no connectionId', () => { + const externalConfig: UserProvidedProxyConfiguration = { + method: 'GET', + providerConfigKey: 'provider-config-key-1', + connectionId: '', + endpoint: 'https://example.com' + }; + const internalConfig: InternalProxyConfiguration = { + provider: 'provider-1', + connection: { + environment_id: 1, + connection_id: 'connection-1', + provider_config_key: 'provider-config-key-1', + credentials: {} as OAuth2Credentials, + connection_config: {} + }, + existingActivityLogId: 1 + }; + const { success, error, response, activityLogs } = proxyService.configure(externalConfig, internalConfig); + expect(success).toBe(false); + expect(response).toBeNull(); + expect(error).toBeDefined(); + expect(error?.message).toContain('missing_connection_id'); + expect(activityLogs.length).toBe(1); + expect(activityLogs[0]).toMatchObject({ + environment_id: 1, + activity_log_id: 1, + level: 'error' + }); + }); + it('Should fail if no providerConfigKey', () => { + const externalConfig: UserProvidedProxyConfiguration = { + method: 'GET', + providerConfigKey: '', + connectionId: 'connection-1', + endpoint: 'https://example.com' + }; + const internalConfig: InternalProxyConfiguration = { + provider: 'provider-1', + connection: { + environment_id: 1, + connection_id: 'connection-1', + provider_config_key: 'provider-config-key-1', + credentials: {} as OAuth2Credentials, + connection_config: {} + }, + existingActivityLogId: 1 + }; + const { success, error, response, activityLogs } = proxyService.configure(externalConfig, internalConfig); + expect(success).toBe(false); + expect(response).toBeNull(); + expect(error).toBeDefined(); + expect(error?.message).toContain('missing_provider_config_key'); + expect(activityLogs.length).toBe(1); + expect(activityLogs[0]).toMatchObject({ + environment_id: 1, + activity_log_id: 1, + level: 'error' + }); + }); + it('Should fail if unknown provider', () => { + const externalConfig: UserProvidedProxyConfiguration = { + method: 'GET', + providerConfigKey: 'provider-config-key-1', + connectionId: 'connection-1', + endpoint: 'https://example.com' + }; + const internalConfig: InternalProxyConfiguration = { + provider: 'unknown', + connection: { + environment_id: 1, + connection_id: 'connection-1', + provider_config_key: 'provider-config-key-1', + credentials: {} as OAuth2Credentials, + connection_config: {} + }, + existingActivityLogId: 1 + }; + const { success, error, response, activityLogs } = proxyService.configure(externalConfig, internalConfig); + expect(success).toBe(false); + expect(response).toBeNull(); + expect(error).toBeDefined(); + expect(error?.message).toContain('proxy is not supported'); + expect(activityLogs.length).toBe(3); + expect(activityLogs[2]).toMatchObject({ + environment_id: 1, + activity_log_id: 1, + level: 'error' + }); + }); + it('Should succeed', () => { + const externalConfig: UserProvidedProxyConfiguration = { + method: 'GET', + providerConfigKey: 'provider-config-key-1', + connectionId: 'connection-1', + endpoint: '/api/test', + retries: 3, + baseUrlOverride: 'https://api.github.com.override', + headers: { + 'x-custom': 'custom-value' + }, + params: { foo: 'bar' }, + responseType: 'blob' + }; + const internalConfig: InternalProxyConfiguration = { + provider: 'github', + connection: { + environment_id: 1, + connection_id: 'connection-1', + provider_config_key: 'provider-config-key-1', + credentials: {} as OAuth2Credentials, + connection_config: {} + }, + existingActivityLogId: 1 + }; + const { success, error, response, activityLogs } = proxyService.configure(externalConfig, internalConfig); + expect(success).toBe(true); + expect(response).toMatchObject({ + endpoint: '/api/test', + method: 'GET', + template: { + auth_mode: 'OAUTH2', + authorization_url: 'https://github.com/login/oauth/authorize', + token_url: 'https://github.com/login/oauth/access_token', + proxy: { + base_url: 'https://api.github.com' + }, + docs: 'https://docs.github.com/en/rest' + }, + token: '', + provider: 'github', + providerConfigKey: 'provider-config-key-1', + connectionId: 'connection-1', + headers: { + 'x-custom': 'custom-value' + }, + retries: 3, + baseUrlOverride: 'https://api.github.com.override', + decompress: false, + connection: { + environment_id: 1, + connection_id: 'connection-1', + provider_config_key: 'provider-config-key-1', + credentials: {}, + connection_config: {} + }, + params: { foo: 'bar' }, + responseType: 'blob' + }); + expect(error).toBeNull(); + expect(activityLogs.length).toBe(4); + }); +}); diff --git a/packages/shared/lib/utils/error.ts b/packages/shared/lib/utils/error.ts index f70acca55a..97edcc0753 100644 --- a/packages/shared/lib/utils/error.ts +++ b/packages/shared/lib/utils/error.ts @@ -491,7 +491,7 @@ export class NangoError extends Error { default: this.status = 500; this.type = 'unhandled_' + type; - this.message = `An unhandled error ${this.payload} has occurred: ${type}`; + this.message = `An unhandled error of type '${type}' with payload '${JSON.stringify(this.payload)}' has occured`; } }