From 7229b490311a45b24c060a5177fb26aea496999f Mon Sep 17 00:00:00 2001
From: Khaliq <khaliq@nango.dev>
Date: Fri, 1 Mar 2024 11:53:38 +0200
Subject: [PATCH] [NAN-478] update runner logic to allow to connect to a remote
 instance (#1768)

## Describe your changes
* Add a remote instance runner so jobs doesn't need to spawn a process.
In some cases I saw that a sync failed because spawning a process took
too long. For enterprise customers they can setup a runner instance and
so in that case jobs will connect to that remote instance instead.
Customers can set this using the `RUNNER_SERVICE_URL` env variable
* Best practice is to use an AWS role for connecting to a bucket so this
update allows customers to configure bucket connection using the role as
long as they have `AWS_REGION` and `AWS_BUCKET_NAME` set

## Issue ticket number and link
NAN-478

## Checklist before requesting a review (skip if just adding/editing
APIs & templates)
- [ ] I added tests, otherwise the reason is:
- [ ] I added observability, otherwise the reason is:
- [ ] I added analytics, otherwise the reason is:
---
 packages/jobs/lib/runner/remote.runner.ts     | 30 +++++++++++++++++++
 packages/jobs/lib/runner/runner.ts            | 18 +++++++++--
 .../lib/services/file/remote.service.ts       |  4 +--
 3 files changed, 47 insertions(+), 5 deletions(-)
 create mode 100644 packages/jobs/lib/runner/remote.runner.ts

diff --git a/packages/jobs/lib/runner/remote.runner.ts b/packages/jobs/lib/runner/remote.runner.ts
new file mode 100644
index 00000000000..2b94ef24183
--- /dev/null
+++ b/packages/jobs/lib/runner/remote.runner.ts
@@ -0,0 +1,30 @@
+import type { Runner } from './runner.js';
+import { RunnerType } from './runner.js';
+import { getRunnerClient } from '@nangohq/nango-runner';
+
+export class RemoteRunner implements Runner {
+    public client: any;
+    public runnerType: RunnerType = RunnerType.Remote;
+    constructor(
+        public readonly id: string,
+        public readonly url: string
+    ) {
+        this.client = getRunnerClient(this.url);
+    }
+
+    async suspend(): Promise<void> {
+        console.warn('cannot suspend a remote runner');
+    }
+
+    toJSON() {
+        return { runnerType: this.runnerType, id: this.id, url: this.url };
+    }
+
+    static fromJSON(obj: any): RemoteRunner {
+        throw new Error(`'fromJSON(${obj})' not implemented`);
+    }
+
+    static async getOrStart(runnerId: string): Promise<RemoteRunner> {
+        return new RemoteRunner(runnerId, process.env['RUNNER_SERVICE_URL'] || 'http://nango-runner');
+    }
+}
diff --git a/packages/jobs/lib/runner/runner.ts b/packages/jobs/lib/runner/runner.ts
index 38bf9ddb85c..00455f8536f 100644
--- a/packages/jobs/lib/runner/runner.ts
+++ b/packages/jobs/lib/runner/runner.ts
@@ -1,12 +1,14 @@
 import type { KVStore } from '@nangohq/shared/lib/utils/kvstore/KVStore.js';
 import { LocalRunner } from './local.runner.js';
 import { RenderRunner } from './render.runner.js';
-import { getEnv, getRedisUrl, InMemoryKVStore, RedisKVStore } from '@nangohq/shared';
+import { RemoteRunner } from './remote.runner.js';
+import { getEnv, getRedisUrl, InMemoryKVStore, RedisKVStore, isEnterprise } from '@nangohq/shared';
 import type { ProxyAppRouter } from '@nangohq/nango-runner';
 
 export enum RunnerType {
     Local = 'local',
-    Render = 'render'
+    Render = 'render',
+    Remote = 'remote'
 }
 
 export interface Runner {
@@ -48,7 +50,15 @@ export async function getOrStartRunner(runnerId: string): Promise<Runner> {
         } catch (err) {}
     }
     const isRender = process.env['IS_RENDER'] === 'true';
-    const runner = isRender ? await RenderRunner.getOrStart(runnerId) : await LocalRunner.getOrStart(runnerId);
+    let runner: Runner;
+    if (isEnterprise()) {
+        runner = await RemoteRunner.getOrStart(runnerId);
+    } else if (isRender) {
+        runner = await RenderRunner.getOrStart(runnerId);
+    } else {
+        runner = await LocalRunner.getOrStart(runnerId);
+    }
+
     await waitForRunner(runner);
     await runnersCache.set(runner);
     return runner;
@@ -86,6 +96,8 @@ class RunnerCache {
                         return LocalRunner.fromJSON(obj);
                     case RunnerType.Render:
                         return RenderRunner.fromJSON(obj);
+                    case RunnerType.Remote:
+                        return RemoteRunner.fromJSON(obj);
                 }
             }
             return undefined;
diff --git a/packages/shared/lib/services/file/remote.service.ts b/packages/shared/lib/services/file/remote.service.ts
index cabd8a70405..494e9f25cee 100644
--- a/packages/shared/lib/services/file/remote.service.ts
+++ b/packages/shared/lib/services/file/remote.service.ts
@@ -13,8 +13,8 @@ import localFileService from './local.service.js';
 let client: S3Client | null = null;
 let useS3 = !isLocal();
 
-if (isEnterprise() && !(process.env['AWS_PUBLIC_ACCESS_KEY_ID'] && process.env['AWS_PUBLIC_SECRET_ACCESS'])) {
-    useS3 = false;
+if (isEnterprise()) {
+    useS3 = Boolean(process.env['AWS_REGION'] && process.env['AWS_BUCKET_NAME']);
     client = new S3Client({
         region: (process.env['AWS_REGION'] as string) || 'us-west-2'
     });