Merge branch 'master' into sam/24_11_27/feat/get-integration-credentials

packages/database/lib/migrations/20241122212401_integration_backfill_missing_fields.cjs

@@ -27,6 +27,7 @@ exports.up = async function (knex) {
         .from('_nango_configs')
         .whereIn('provider', clientIdProviders)
         .whereRaw("NOT (missing_fields @> '{oauth_client_id}')")
+        .where('oauth_client_id', null)
         .update({ missing_fields: knex.raw("array_append(missing_fields, 'oauth_client_id')") });
 
     const needsClientSecret = ['OAUTH1', 'OAUTH2', 'TBA', 'APP'];
@@ -38,6 +39,7 @@ exports.up = async function (knex) {
         .from('_nango_configs')
         .whereIn('provider', clientSecretProviders)
         .whereRaw("NOT (missing_fields @> '{oauth_client_secret}')")
+        .where('oauth_client_secret', null)
         .update({ missing_fields: knex.raw("array_append(missing_fields, 'oauth_client_secret')") });
 
     const needsAppLink = ['APP'];
@@ -49,6 +51,7 @@ exports.up = async function (knex) {
         .from('_nango_configs')
         .whereIn('provider', appLinkProviders)
         .whereRaw("NOT (missing_fields @> '{app_link}')")
+        .where('app_link', null)
         .update({ missing_fields: knex.raw("array_append(missing_fields, 'app_link')") });
 };
 

packages/server/lib/controllers/sync/deploy/postConfirmation.integration.test.ts

@@ -70,7 +70,62 @@ describe(`POST ${endpoint}`, () => {
             deletedSyncs: [],
             newActions: [],
             newSyncs: [],
-            deletedModels: []
+            deletedModels: [],
+            newOnEventScripts: [],
+            deletedOnEventScripts: []
+        });
+        expect(res.res.status).toBe(200);
+    });
+
+    it('should show correct on-events scripts diff', async () => {
+        const { account, env: environment } = await seeders.seedAccountEnvAndUser();
+        const { unique_key: providerConfigKey } = await seeders.createConfigSeed(environment, 'notion-123', 'notion');
+        const existingOnEvent = await seeders.createOnEventScript({ account, environment, providerConfigKey });
+
+        const res = await api.fetch(endpoint, {
+            method: 'POST',
+            token: environment.secret_key,
+            body: {
+                debug: false,
+                flowConfigs: [],
+                onEventScriptsByProvider: [
+                    {
+                        providerConfigKey,
+                        scripts: [
+                            {
+                                name: 'new-script',
+                                event: 'post-connection-creation',
+                                fileBody: { js: '', ts: '' }
+                            }
+                        ]
+                    }
+                ],
+                reconcile: false
+            }
+        });
+
+        isSuccess(res.json);
+
+        expect(res.json).toStrictEqual<typeof res.json>({
+            deletedActions: [],
+            deletedSyncs: [],
+            newActions: [],
+            newSyncs: [],
+            deletedModels: [],
+            newOnEventScripts: [
+                {
+                    name: 'new-script',
+                    providerConfigKey,
+                    event: 'post-connection-creation'
+                }
+            ],
+            deletedOnEventScripts: [
+                {
+                    name: existingOnEvent.name,
+                    providerConfigKey: existingOnEvent.providerConfigKey,
+                    event: existingOnEvent.event
+                }
+            ]
         });
         expect(res.res.status).toBe(200);
     });

packages/server/lib/controllers/sync/deploy/postConfirmation.ts

@@ -1,7 +1,7 @@
 import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils';
-import type { PostDeployConfirmation } from '@nangohq/types';
+import type { PostDeployConfirmation, ScriptDifferences } from '@nangohq/types';
 import { asyncWrapper } from '../../../utils/asyncWrapper.js';
-import { getAndReconcileDifferences } from '@nangohq/shared';
+import { getAndReconcileDifferences, onEventScriptService } from '@nangohq/shared';
 import { getOrchestrator } from '../../../utils/utils.js';
 import { logContextGetter } from '@nangohq/logs';
 import { validation } from './validation.js';
@@ -24,7 +24,7 @@ export const postDeployConfirmation = asyncWrapper<PostDeployConfirmation>(async
     const body: PostDeployConfirmation['Body'] = val.data;
     const environmentId = res.locals['environment'].id;
 
-    const result = await getAndReconcileDifferences({
+    const syncAndActionDifferences = await getAndReconcileDifferences({
         environmentId,
         flows: body.flowConfigs,
         performAction: false,
@@ -33,10 +33,41 @@ export const postDeployConfirmation = asyncWrapper<PostDeployConfirmation>(async
         logContextGetter,
         orchestrator
     });
-    if (!result) {
+    if (!syncAndActionDifferences) {
         res.status(500).send({ error: { code: 'server_error' } });
         return;
     }
 
+    let result: ScriptDifferences;
+    if (body.onEventScriptsByProvider) {
+        const diff = await onEventScriptService.diffChanges({
+            environmentId,
+            onEventScriptsByProvider: body.onEventScriptsByProvider
+        });
+        result = {
+            ...syncAndActionDifferences,
+            newOnEventScripts: diff.added.map((script) => {
+                return {
+                    providerConfigKey: script.providerConfigKey,
+                    name: script.name,
+                    event: script.event
+                };
+            }),
+            deletedOnEventScripts: diff.deleted.map((script) => {
+                return {
+                    providerConfigKey: script.providerConfigKey,
+                    name: script.name,
+                    event: script.event
+                };
+            })
+        };
+    } else {
+        result = {
+            ...syncAndActionDifferences,
+            newOnEventScripts: [],
+            deletedOnEventScripts: []
+        };
+    }
+
     res.status(200).send(result);
 });

packages/shared/lib/seeders/index.ts

@@ -6,3 +6,4 @@ export * from './global.seeder.js';
 export * from './sync-job.seeder.js';
 export * from './sync.seeder.js';
 export * from './user.seeder.js';
+export * from './onEventScript.seeder.js';

packages/shared/lib/seeders/onEventScript.seeder.ts

@@ -0,0 +1,34 @@
+import type { DBEnvironment, DBTeam, OnEventScript, OnEventScriptsByProvider } from '@nangohq/types';
+import { onEventScriptService } from '../services/on-event-scripts.service.js';
+
+export async function createOnEventScript({
+    account,
+    environment,
+    providerConfigKey
+}: {
+    account: DBTeam;
+    environment: DBEnvironment;
+    providerConfigKey: string;
+}): Promise<OnEventScript> {
+    const scripts: OnEventScriptsByProvider[] = [
+        {
+            providerConfigKey,
+            scripts: [
+                {
+                    name: 'test-script',
+                    event: 'post-connection-creation',
+                    fileBody: { js: '', ts: '' }
+                }
+            ]
+        }
+    ];
+    const [added] = await onEventScriptService.update({
+        environment,
+        account,
+        onEventScriptsByProvider: scripts
+    });
+    if (!added) {
+        throw new Error('failed_to_create_on_event_script');
+    }
+    return added;
+}

packages/shared/lib/services/on-event-scripts.service.ts

@@ -1,20 +1,60 @@
 import db from '@nangohq/database';
 import remoteFileService from './file/remote.service.js';
 import { env } from '@nangohq/utils';
-import type { OnEventScriptsByProvider, OnEventScript, DBTeam, DBEnvironment, OnEventType } from '@nangohq/types';
+import type { OnEventScriptsByProvider, DBOnEventScript, DBTeam, DBEnvironment, OnEventType, OnEventScript } from '@nangohq/types';
 import { increment } from './sync/config/config.service.js';
 import configService from './config.service.js';
 
 const TABLE = 'on_event_scripts';
 
-function toDbEvent(eventType: OnEventType): OnEventScript['event'] {
-    switch (eventType) {
-        case 'post-connection-creation':
-            return 'POST_CONNECTION_CREATION';
-        case 'pre-connection-deletion':
-            return 'PRE_CONNECTION_DELETION';
+const EVENT_TYPE_MAPPINGS: Record<DBOnEventScript['event'], OnEventType> = {
+    POST_CONNECTION_CREATION: 'post-connection-creation',
+    PRE_CONNECTION_DELETION: 'pre-connection-deletion'
+} as const;
+
+const eventTypeMapper = {
+    fromDb: (event: DBOnEventScript['event']): OnEventType => {
+        return EVENT_TYPE_MAPPINGS[event];
+    },
+    toDb: (eventType: OnEventType): DBOnEventScript['event'] => {
+        for (const [key, value] of Object.entries(EVENT_TYPE_MAPPINGS)) {
+            if (value === eventType) {
+                return key as DBOnEventScript['event'];
+            }
+        }
+        throw new Error(`Unknown event type: ${eventType}`); // This should never happen
+    }
+};
+
+const dbMapper = {
+    to: (script: OnEventScript): DBOnEventScript => {
+        return {
+            id: script.id,
+            config_id: script.configId,
+            name: script.name,
+            file_location: script.fileLocation,
+            version: script.version,
+            active: script.active,
+            event: eventTypeMapper.toDb(script.event),
+            created_at: script.createdAt,
+            updated_at: script.updatedAt
+        };
+    },
+    from: (dbScript: DBOnEventScript & { provider_config_key: string }): OnEventScript => {
+        return {
+            id: dbScript.id,
+            configId: dbScript.config_id,
+            providerConfigKey: dbScript.provider_config_key,
+            name: dbScript.name,
+            fileLocation: dbScript.file_location,
+            version: dbScript.version,
+            active: dbScript.active,
+            event: eventTypeMapper.fromDb(dbScript.event),
+            createdAt: dbScript.created_at,
+            updatedAt: dbScript.updated_at
+        };
     }
-}
+};
 
 export const onEventScriptService = {
     async update({
@@ -25,14 +65,14 @@ export const onEventScriptService = {
         environment: DBEnvironment;
         account: DBTeam;
         onEventScriptsByProvider: OnEventScriptsByProvider[];
-    }): Promise<(OnEventScript & { providerConfigKey: string })[]> {
+    }): Promise<OnEventScript[]> {
         return db.knex.transaction(async (trx) => {
-            const onEventInserts: Omit<OnEventScript, 'id' | 'created_at' | 'updated_at'>[] = [];
+            const onEventInserts: Omit<DBOnEventScript, 'id' | 'created_at' | 'updated_at'>[] = [];
 
             // Deactivate all previous scripts for the environment
             // This is done to ensure that we don't have any orphaned scripts when they are removed from nango.yaml
             const previousScriptVersions = await trx
-                .from<OnEventScript>(TABLE)
+                .from<DBOnEventScript>(TABLE)
                 .whereRaw(`config_id IN (SELECT id FROM _nango_configs WHERE environment_id = ?)`, [environment.id])
                 .where({
                     active: true
@@ -52,7 +92,7 @@ export const onEventScriptService = {
 
                 for (const script of scripts) {
                     const { name, fileBody, event: scriptEvent } = script;
-                    const event = toDbEvent(scriptEvent);
+                    const event = eventTypeMapper.toDb(scriptEvent);
 
                     const previousScriptVersion = previousScriptVersions.find((p) => p.config_id === config.id && p.name === name && p.event === event);
                     const version = previousScriptVersion ? increment(previousScriptVersion.version) : '0.0.1';
@@ -84,20 +124,88 @@ export const onEventScriptService = {
                 }
             }
             if (onEventInserts.length > 0) {
-                type R = Awaited<ReturnType<typeof onEventScriptService.update>>;
                 const res = await trx
                     .with('inserted', (qb) => {
                         qb.insert(onEventInserts).into(TABLE).returning('*');
                     })
-                    .select<R>(['inserted.*', '_nango_configs.unique_key as providerConfigKey'])
+                    .select<(DBOnEventScript & { provider_config_key: string })[]>(['inserted.*', '_nango_configs.unique_key as provider_config_key'])
                     .from('inserted')
                     .join('_nango_configs', 'inserted.config_id', '_nango_configs.id');
-                return res;
+                return res.map(dbMapper.from);
             }
             return [];
         });
     },
-    getByConfig: async (configId: number, event: OnEventType): Promise<OnEventScript[]> => {
-        return db.knex.from<OnEventScript>(TABLE).where({ config_id: configId, active: true, event: toDbEvent(event) });
+
+    getByEnvironmentId: async (environmentId: number): Promise<OnEventScript[]> => {
+        const existingScriptsQuery = await db.knex
+            .select<(DBOnEventScript & { provider_config_key: string })[]>(`${TABLE}.*`, '_nango_configs.unique_key as provider_config_key')
+            .from(TABLE)
+            .join('_nango_configs', `${TABLE}.config_id`, '_nango_configs.id')
+            .where({
+                '_nango_configs.environment_id': environmentId,
+                [`${TABLE}.active`]: true
+            });
+        return existingScriptsQuery.map(dbMapper.from);
+    },
+
+    getByConfig: async (configId: number, event: OnEventType): Promise<DBOnEventScript[]> => {
+        return db.knex.from<DBOnEventScript>(TABLE).where({ config_id: configId, active: true, event: eventTypeMapper.toDb(event) });
+    },
+
+    diffChanges: async ({
+        environmentId,
+        onEventScriptsByProvider
+    }: {
+        environmentId: number;
+        onEventScriptsByProvider: OnEventScriptsByProvider[];
+    }): Promise<{
+        added: Omit<OnEventScript, 'id' | 'fileLocation' | 'createdAt' | 'updatedAt'>[];
+        deleted: OnEventScript[];
+        updated: OnEventScript[];
+    }> => {
+        const res: Awaited<ReturnType<typeof onEventScriptService.diffChanges>> = {
+            added: [],
+            deleted: [],
+            updated: []
+        };
+
+        const existingScripts = await onEventScriptService.getByEnvironmentId(environmentId);
+
+        // Create a map of existing scripts for easier lookup
+        const previousMap = new Map(existingScripts.map((script) => [`${script.configId}:${script.name}:${script.event}`, script]));
+
+        for (const provider of onEventScriptsByProvider) {
+            const config = await configService.getProviderConfig(provider.providerConfigKey, environmentId);
+            if (!config || !config.id) continue;
+
+            for (const script of provider.scripts) {
+                const key = `${config.id}:${script.name}:${script.event}`;
+
+                const maybeScript = previousMap.get(key);
+                if (maybeScript) {
+                    // Script already exists - it's an update
+                    res.updated.push(maybeScript);
+
+                    // Remove from map to track deletions
+                    previousMap.delete(key);
+                } else {
+                    // Script doesn't exist - it's new
+                    res.added.push({
+                        configId: config.id,
+                        name: script.name,
+                        version: '0.0.1',
+                        active: true,
+                        event: script.event,
+                        providerConfigKey: provider.providerConfigKey
+                    });
+                }
+            }
+        }
+
+        // Any remaining scripts in the map were not found - they are deleted
+        res.deleted.push(...Array.from(previousMap.values()));
+
+        return res;
     }
 };