Add persist API

The goal is to remove the database dependency for the runner. Writing customers data (batchSave/.../activityLogs/lastSyncDate) will go through this API
NangoHQ · Jan 30, 2024 · 7cfb5fe · 7cfb5fe
1 parent 6fce964
commit 7cfb5fe
Show file tree

Hide file tree

Showing 16 changed files with 876 additions and 16 deletions.
diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml
@@ -4,6 +4,13 @@ concurrency:
     group: pulls/${{ github.event.pull_request.number || github.ref }}
     cancel-in-progress: true
 jobs:
+    nango-persist:
+        uses: ./.github/workflows/push-container.yaml
+        secrets: inherit
+        with:
+            package: persist
+            run-cmd: ts-build
+            tags: -t nangohq/nango-persist:$GITHUB_SHA ${{ github.ref == 'refs/heads/master' && '-t nangohq/nango-persist:enterprise-$GITHUB_SHA -t nangohq/nango-persist:enterprise -t nangohq/nango-persist:hosted-$GITHUB_SHA -t nangohq/nango-persist:hosted' || '' }}
     nango-jobs:
         uses: ./.github/workflows/push-container.yaml
         secrets: inherit

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -20,6 +20,7 @@ on:
                     - server
                     - jobs
                     - runner
+                    - persist
 
 jobs:
     deploy_server:
@@ -76,4 +77,20 @@ jobs:
                 RUNNER_OWNER_ID: ${{ secrets.RENDER_RUNNER_OWNER_ID }}
               run: |
                 bash ./scripts/deploy/runners.bash
-
+    depoy_persist:
+        if: inputs.service == 'persist'
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKER_USERNAME }}
+                password: ${{ secrets.DOCKER_PASSWORD }}
+            - name: Push tag
+              run: |
+                docker buildx imagetools create nangohq/nango-persist:${{ github.sha }} --tag nangohq/nango-persist:${{ inputs.stage }}
+            - name: Deploy persist
+              run: |
+                SERVICE_ID=${{ fromJson('{ production: "srv-XXXXXXXXXXXXX", staging: "srv-XXXXXXXXXXXXXXXX" }')[inputs.stage] }}
+                curl --request POST "https://api.render.com/v1/services/$SERVICE_ID/deploys" --header "authorization: Bearer ${{ secrets.RENDER_API_KEY }}"
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -8,6 +8,7 @@
         "packages/node-client",
         "packages/server",
         "packages/runner",
+        "packages/persist",
         "packages/jobs"
     ],
     "scripts": {
@@ -18,7 +19,7 @@
         "lint": "eslint . --ext .ts,.tsx",
         "lint:fix": "eslint . --ext .ts,.tsx --fix",
         "install:all": "npm i && cd packages/webapp && npm i && cd ../..",
-        "ts-build": "tsc -b --clean packages/shared packages/server packages/cli packages/runner packages/jobs && tsc -b tsconfig.build.json && npm run postbuild -ws --if-present && tsc -b packages/webapp/tsconfig.json",
+        "ts-build": "tsc -b --clean packages/shared packages/server packages/cli packages/runner packages/jobs packages/persist && tsc -b tsconfig.build.json && npm run postbuild -ws --if-present && tsc -b packages/webapp/tsconfig.json",
         "docker-build": "docker build -f packages/server/Dockerfile -t nango-server:latest .",
         "webapp-build:hosted": "cd ./packages/webapp && npm run build:hosted && cd ../..",
         "webapp-build:staging": "cd ./packages/webapp && npm run build:staging && cd ../..",
@@ -33,13 +34,14 @@
         "build:enterprise": "npm run install:all && npm run ts-build && npm run webapp-build:enterprise",
         "server:dev:watch": "cd ./packages/server && npm run dev",
         "jobs:dev:watch": "npm run dev -w @nangohq/nango-jobs",
+        "persist:dev:watch": "npm run dev -w @nangohq/nango-persist",
         "webapp:dev:watch": "cd ./packages/webapp && npm run start:hosted",
         "cli:watch": "cd ./packages/cli && npm run dev",
         "prepare": "husky install",
         "build:watch": "tsc -b -w tsconfig.build.json",
         "dev:watch": "concurrently --kill-others \"npm run build:watch\" \"npm run webapp-build:watch\" \"npm run prettier-watch\" \"npm run shared:watch\" \"npm run cli:watch\"",
         "watch:dev": "npm run dev:watch",
-        "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\" \"npm run jobs:dev:watch\"",
+        "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\" \"npm run jobs:dev:watch\" \"npm run persist:dev:watch\"",
         "watch:dev:apps": "npm run dev:watch:apps",
         "dw": "npm run dev:watch",
         "dwa": "npm run dev:watch:apps",

diff --git a/packages/persist/.gitignore b/packages/persist/.gitignore
@@ -0,0 +1,3 @@
+tsconfig.tsbuildinfo
+dist/*
+node_modules