Merge pull request #1226 from rudderlabs/release/v1.65.0

chore(release): pull release/v1.65.0 into main

.github/workflows/commitlint.yml

@@ -0,0 +1,36 @@
+name: Commitlint
+
+on: [push]
+
+jobs:
+  commitlint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Print versions
+        run: |
+          git --version
+          node --version
+          npm --version
+          npx commitlint --version
+
+      # Run the commitlint action, considering its own dependencies and yours as well 🚀
+      # `github.workspace` is the path to your repository.
+      - uses: wagoid/commitlint-github-action@v5
+        env:
+          NODE_PATH: ${{ github.workspace }}/node_modules
+        with:
+          commitDepth: 1

.github/workflows/deploy.yml

@@ -43,22 +43,20 @@ jobs:
         run: |
           npm ci
 
-      - name: Execute Unit Tests
-        env:
-          HUSKY: 0
-        run: |
-          npm run test:ci
-
-      - name: Install Python
-        uses: actions/[email protected]
-        with:
-          python-version: '3.9.13'
+      - name: Set up Python
+        run: scripts/setup-python.sh
 
       - name: Display Python Version
         run: |
           which python
           python -c "import sys; print(sys.version)"
 
+      - name: Execute Unit Tests
+        env:
+          HUSKY: 0
+        run: |
+          npm run test:ci
+
       - name: Install Python Dependencies
         run: pip3 install -r ./scripts/requirements.txt
 

.github/workflows/report-code-coverage.yml

@@ -27,6 +27,9 @@ jobs:
       - name: Install Dependencies
         run: npm ci
 
+      - name: Set up Python
+        run: scripts/setup-python.sh
+
       - name: Run Tests
         run: npm run test:ci
 

.github/workflows/test.yml

@@ -26,12 +26,12 @@ jobs:
       - name: Install Dependencies
         run: npm ci
 
-      - name: Run Unit Tests
-        run: npm run test:ci
-
       - name: Set up Python
         run: scripts/setup-python.sh
 
+      - name: Run Tests
+        run: npm run test:ci
+
       - name: Get changed files
         id: changed_files
         uses: tj-actions/changed-files@v41

.github/workflows/verify.yml

@@ -0,0 +1,53 @@
+name: Code quality checks
+
+on:
+  pull_request:
+
+jobs:
+  py-scripts-lint:
+    name: Check for formatting of python scripts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Setting up python libraries
+        run: ./scripts/setup-python.sh
+
+      # Reference: https://black.readthedocs.io/en/stable/integrations/github_actions.html
+      - name: Check formatting for Python files
+        uses: psf/black@stable
+        with:
+          options: '--check --verbose'
+
+  formatting-lint:
+    name: Check for formatting & lint errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: .nvmrc
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Run Lint Checks
+        run: |
+          npm run lint
+
+      - run: git diff --exit-code
+
+      - name: Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'ESLint check is failing. Please run `npm run lint` on your working copy and commit the changes.'

.gitignore

@@ -30,3 +30,5 @@ git
 reports
 .eslintcache
 .vscode
+
+__pycache__

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
 
-npm run pre-commit
+npm run pre-commit

.nvmrc

@@ -1 +1 @@
-18.15.0
+20.11.0

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.65.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.2...v1.65.0) (2024-02-20)
+
+
+### Features
+
+* add connectionMode for all sourceTypes in dest def for selected destinations ([#1197](https://github.com/rudderlabs/rudder-config-schema/issues/1197)) ([dde686c](https://github.com/rudderlabs/rudder-config-schema/commit/dde686cc72e0ac7423487d566e8f9715cc43d24f))
+* add GCM to FB Pixel, GA, GMT, HobSpot, Redis, S3, Slack, VWO ([#1190](https://github.com/rudderlabs/rudder-config-schema/issues/1190)) ([f40d5cd](https://github.com/rudderlabs/rudder-config-schema/commit/f40d5cda403e39e149f132dc1ffc2d1efa50a186))
+* enabled sql model support for mysql ([#1222](https://github.com/rudderlabs/rudder-config-schema/issues/1222)) ([09bbe2a](https://github.com/rudderlabs/rudder-config-schema/commit/09bbe2a5d1f81c26dec71582fbb7069550e7c7af))
+* improve schema generator output ([#1191](https://github.com/rudderlabs/rudder-config-schema/issues/1191)) ([04e05b8](https://github.com/rudderlabs/rudder-config-schema/commit/04e05b89a7333291792fbb6e111388b5ff8b985f))
+* onboard new destination commandbar ([#1196](https://github.com/rudderlabs/rudder-config-schema/issues/1196)) ([daf9fa8](https://github.com/rudderlabs/rudder-config-schema/commit/daf9fa8256eed280bad04c0c109de3787e6262e0))
+* onboard trade desk real time conversions ([#1213](https://github.com/rudderlabs/rudder-config-schema/issues/1213)) ([0d9b8d7](https://github.com/rudderlabs/rudder-config-schema/commit/0d9b8d7ee45b6ba5c914aaca6c2c11474a7575e9))
+* onboarding bluecore integration ([#1182](https://github.com/rudderlabs/rudder-config-schema/issues/1182)) ([6d5e40f](https://github.com/rudderlabs/rudder-config-schema/commit/6d5e40fdb812aa4c46ea13cbae5f7eb696d0cc0c))
+* tiktok_offline_events added support for  all Standard  events ([#1216](https://github.com/rudderlabs/rudder-config-schema/issues/1216)) ([61d36fa](https://github.com/rudderlabs/rudder-config-schema/commit/61d36fa5db882f00d97dd9dc88b2f26a7d49a87a))
+
+
+### Bug Fixes
+
+* add support of placing properties at root in af ([#1203](https://github.com/rudderlabs/rudder-config-schema/issues/1203)) ([1859354](https://github.com/rudderlabs/rudder-config-schema/commit/185935467899222b8766c63c63b934d70410cc9a))
+* comscore schema ([#1217](https://github.com/rudderlabs/rudder-config-schema/issues/1217)) ([7faa64a](https://github.com/rudderlabs/rudder-config-schema/commit/7faa64acfe2fca20b75789f31b541dd33e7fde8c))
+
 ### [1.64.2](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.1...v1.64.2) (2024-02-14)
 
 ### [1.64.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.0...v1.64.1) (2024-02-13)

migration/migrate.py

@@ -2,116 +2,126 @@
 import json
 import os
 
-DEST_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/destinations/schemas/'
-DEST_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/destinationConfigs/'
-DEST_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/'
+DEST_WEBAPP_FILE_PATH = "rudder-webapp/src/components/destinations/schemas/"
+DEST_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/destinationConfigs/"
+DEST_SCHEMA_FILE_PATH = "rudder-config-backend/src/scripts/schemaList/destinations/"
 
-SOURCE_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/'
-SOURCE_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/sourceConfigs/'
+SOURCE_WEBAPP_FILE_PATH = (
+    "rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/"
+)
+SOURCE_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/sourceConfigs/"
 # SOURCE_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/'
 
-CONFIGURATIONS_DIR_PATH = '../src/configurations'
-DEST_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/destinations'
-SRC_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/sources'
+CONFIGURATIONS_DIR_PATH = "../src/configurations"
+DEST_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/destinations"
+SRC_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/sources"
+
 
 def update_destination():
-    dest_list = [d[:-5].lower() for d in os.listdir(f'../../{DEST_CONFIG_BE_FILE_PATH}')]
+    dest_list = [
+        d[:-5].lower() for d in os.listdir(f"../../{DEST_CONFIG_BE_FILE_PATH}")
+    ]
     for dest in dest_list:
         final_data = []
         # read db_config
-        with open(f'../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json', 'r') as f:
+        with open(f"../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json", "r") as f:
             db_config = json.loads(f.read())
-        print (db_config)
+        print(db_config)
         final_data.append(db_config)
-        print ("========================")
+        print("========================")
         # read ui_config
-        with open(f'../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json', 'r') as f:
+        with open(f"../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json", "r") as f:
             ui_config = {"uiConfig": json.loads(f.read())}
-        print (ui_config)
+        print(ui_config)
         final_data.append(ui_config)
-        print ("========================")
+        print("========================")
         # read schema
-        if f'{dest.upper()}.json' in os.listdir(f'../../{DEST_SCHEMA_FILE_PATH}'):
-            with open(f'../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json', 'r') as f:
+        if f"{dest.upper()}.json" in os.listdir(f"../../{DEST_SCHEMA_FILE_PATH}"):
+            with open(f"../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json", "r") as f:
                 schema = {"schema": json.loads(f.read())}
         else:
             schema = {"schema": None}
-        print (schema)
+        print(schema)
         final_data.append(schema)
-        print ("========================")
+        print("========================")
         # metadata
-        with open('dest_meta_template.json', 'r') as f:
+        with open("dest_meta_template.json", "r") as f:
             metadata = json.loads(f.read())
-        print (metadata)
+        print(metadata)
         final_data.append(metadata)
-        print ("========================")
+        print("========================")
 
         ########################
         ## write new files
         ########################
         if dest not in os.listdir(DEST_CONFIG_PATH):
-            os.system(f'mkdir {DEST_CONFIG_PATH}/{dest}')
-            print (f'created directory for {dest}')
+            os.system(f"mkdir {DEST_CONFIG_PATH}/{dest}")
+            print(f"created directory for {dest}")
 
-        file_names = ['db-config', 'ui-config', 'schema', 'metadata']
+        file_names = ["db-config", "ui-config", "schema", "metadata"]
         for index, f_name in enumerate(file_names):
-            print (f'writing {f_name} for {dest}...')
-            with open(f'{DEST_CONFIG_PATH}/{dest}/{f_name}.json', 'w') as f:
+            print(f"writing {f_name} for {dest}...")
+            with open(f"{DEST_CONFIG_PATH}/{dest}/{f_name}.json", "w") as f:
                 f.write(json.dumps(final_data[index], indent=2))
 
-        print (f'complete for {dest}...')
-        print ('------------------------------------------------')
+        print(f"complete for {dest}...")
+        print("------------------------------------------------")
+
 
 def update_source():
-    source_list = [d[:-5].lower() for d in os.listdir(f'../../{SOURCE_CONFIG_BE_FILE_PATH}')]
+    source_list = [
+        d[:-5].lower() for d in os.listdir(f"../../{SOURCE_CONFIG_BE_FILE_PATH}")
+    ]
     for source in source_list:
         final_data = []
         # read db_config
-        with open(f'../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json', 'r') as f:
+        with open(f"../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json", "r") as f:
             db_config = json.loads(f.read())
-        print (db_config)
+        print(db_config)
         final_data.append(db_config)
-        print ("========================")
+        print("========================")
         # read ui_config
-        if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_WEBAPP_FILE_PATH}'):
-            with open(f'../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json', 'r') as f:
+        if f"{source.upper()}.json" in os.listdir(f"../../{SOURCE_WEBAPP_FILE_PATH}"):
+            with open(
+                f"../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json", "r"
+            ) as f:
                 ui_config = {"uiConfig": json.loads(f.read())}
         else:
             ui_config = {"uiConfig": None}
-        print (ui_config)
+        print(ui_config)
         final_data.append(ui_config)
-        print ("========================")
+        print("========================")
         # read schema (we don't need schema for sources)
         # if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_SCHEMA_FILE_PATH}'):
         #     with open(f'../../{SOURCE_SCHEMA_FILE_PATH}{source.upper()}.json', 'r') as f:
         #         schema = {"schema": json.loads(f.read())}
         # else:
         schema = {"schema": None}
-        print (schema)
+        print(schema)
         final_data.append(schema)
-        print ("========================")
+        print("========================")
         # metadata
-        with open('source_meta_template.json', 'r') as f:
+        with open("source_meta_template.json", "r") as f:
             metadata = json.loads(f.read())
-        print (metadata)
+        print(metadata)
         final_data.append(metadata)
-        print ("========================")
+        print("========================")
 
         ########################
         ## write new files
         ########################
         if source not in os.listdir(SRC_CONFIG_PATH):
-            os.system(f'mkdir {SRC_CONFIG_PATH}/{source}')
-            print (f'created directory for {source}')
+            os.system(f"mkdir {SRC_CONFIG_PATH}/{source}")
+            print(f"created directory for {source}")
 
-        file_names = ['db_config', 'ui_config', 'schema', 'metadata']
+        file_names = ["db_config", "ui_config", "schema", "metadata"]
         for index, f_name in enumerate(file_names):
-            print (f'writing {f_name} for {source}...')
-            with open(f'{SRC_CONFIG_PATH}/{source}/{f_name}.json', 'w') as f:
+            print(f"writing {f_name} for {source}...")
+            with open(f"{SRC_CONFIG_PATH}/{source}/{f_name}.json", "w") as f:
                 f.write(json.dumps(final_data[index], indent=2))
 
-        print (f'complete for {source}...')
-        print ('------------------------------------------------')
+        print(f"complete for {source}...")
+        print("------------------------------------------------")
 
 
 if __name__ == "__main__":