From 04e05b89a7333291792fbb6e111388b5ff8b985f Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:12:37 +0530 Subject: [PATCH 01/20] feat: improve schema generator output (#1191) --- .github/workflows/report-code-coverage.yml | 3 + .github/workflows/test.yml | 6 +- .gitignore | 2 + .nvmrc | 2 +- scripts/constants.py | 11 + scripts/deployToDB.py | 4 +- scripts/schemaGenerator.py | 137 +++-- scripts/template-db-config.json | 6 +- scripts/utils.py | 51 ++ .../destinations/am/schema.json | 4 +- .../destinations/am/ui-config.json | 4 +- .../destinations/spotifyPixel/db-config.json | 65 ++- .../destinations/spotifyPixel/schema.json | 145 +++-- .../destinations/spotifyPixel/ui-config.json | 510 +++++++++--------- .../create_new_schema_dest/db-config.json | 44 ++ .../create_new_schema_dest/ui-config.json | 118 ++++ .../db-config.json | 49 ++ .../ui-config.json | 153 ++++++ .../update_schema_dest/db-config.json | 44 ++ .../update_schema_dest/schema.json | 92 ++++ .../update_schema_dest/ui-config.json | 118 ++++ .../db-config.json | 49 ++ .../update_schema_dest_legacy_ui/schema.json | 106 ++++ .../ui-config.json | 153 ++++++ .../data/createNewSchemaDest.json | 100 ++++ .../data/createNewSchemaDestLegacyUI.json | 114 ++++ .../data/updateSchemaDest.json | 100 ++++ .../data/updateSchemaDestLegacyUI.json | 114 ++++ test/component_tests/schemaGenerator.test.ts | 151 ++++++ .../validation/destinations/spotifyPixel.json | 18 +- 30 files changed, 2027 insertions(+), 446 deletions(-) create mode 100644 scripts/constants.py create mode 100644 scripts/utils.py create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json create mode 100644 test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest/db-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest/schema.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest/ui-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json create mode 100644 test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json create mode 100644 test/component_tests/data/createNewSchemaDest.json create mode 100644 test/component_tests/data/createNewSchemaDestLegacyUI.json create mode 100644 test/component_tests/data/updateSchemaDest.json create mode 100644 test/component_tests/data/updateSchemaDestLegacyUI.json create mode 100644 test/component_tests/schemaGenerator.test.ts diff --git a/.github/workflows/report-code-coverage.yml b/.github/workflows/report-code-coverage.yml index 2ff1badab..9791eb5d7 100644 --- a/.github/workflows/report-code-coverage.yml +++ b/.github/workflows/report-code-coverage.yml @@ -26,6 +26,9 @@ jobs: - name: Install Dependencies run: npm ci + + - name: Set up Python + run: scripts/setup-python.sh - name: Run Tests run: npm run test:ci diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8fe4e8e1..e65062640 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,12 +26,12 @@ jobs: - name: Install Dependencies run: npm ci - - name: Run Unit Tests - run: npm run test:ci - - name: Set up Python run: scripts/setup-python.sh + - name: Run Tests + run: npm run test:ci + - name: Get changed files id: changed_files uses: tj-actions/changed-files@v41 diff --git a/.gitignore b/.gitignore index 7b515f143..91868eec6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ git reports .eslintcache .vscode + +__pycache__ diff --git a/.nvmrc b/.nvmrc index 55bffd620..8b0beab16 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18.15.0 +20.11.0 diff --git a/scripts/constants.py b/scripts/constants.py new file mode 100644 index 000000000..54b717202 --- /dev/null +++ b/scripts/constants.py @@ -0,0 +1,11 @@ +import os + +__DEFAULT_CONFIG_DIR = 'src/configurations' +CONFIG_DIR = __DEFAULT_CONFIG_DIR + +if 'CONFIG_DIR' in os.environ: + CONFIG_DIR = os.environ['CONFIG_DIR'] + +TEST_INTEGRATION_NAME_PREFIX = 'test_' + +TEST_INTEGRATION_NAME_SUFFIX = '_ignore' diff --git a/scripts/deployToDB.py b/scripts/deployToDB.py index 897944333..6c513fb92 100644 --- a/scripts/deployToDB.py +++ b/scripts/deployToDB.py @@ -4,6 +4,7 @@ import os import sys import jsondiff +from constants import CONFIG_DIR ######################### # ENV VARIABLES FOT TESTING @@ -27,7 +28,6 @@ # CONSTANTS HEADER = {"Content-Type": "application/json"} AUTH = (USERNAME, PASSWORD) -CONFIG_DIR = 'src/configurations' ######################### @@ -105,7 +105,7 @@ def update_diff_db(selector): # check if item is a directory if not os.path.isdir(f'./{CONFIG_DIR}/{selector}s/{item}'): continue - + updated_data = get_file_content(item, selector) persisted_data = get_config_definition(CONTROL_PLANE_URL, selector, updated_data["name"]) diff --git a/scripts/schemaGenerator.py b/scripts/schemaGenerator.py index 96e590670..36d9d9ee3 100644 --- a/scripts/schemaGenerator.py +++ b/scripts/schemaGenerator.py @@ -9,14 +9,11 @@ 2. python3 scripts/schemaGenerator.py -all source ''' import os -import sys import warnings -import json -from jsondiff import diff from enum import Enum import argparse - -CONFIG_DIR = 'src/configurations' +from utils import get_json_from_file, get_json_diff, apply_json_diff, get_formatted_json +from constants import CONFIG_DIR EXCLUDED_DEST = ['postgres', 'bq', 'azure_synapse', 'clickhouse', 'deltalake', 'kafka'] @@ -134,15 +131,21 @@ def generate_schema_for_default_checkbox(field, dbConfig, schema_field_name): Returns: object """ - defaultCheckboxObj = { - "type": FieldTypeEnum.OBJECT.value, - "properties": { - "web": { - "type": FieldTypeEnum.BOOLEAN.value - } - } - } - return defaultCheckboxObj + isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + defaultCheckboxSchemaObj = {} + if isSourceDependent: + defaultCheckboxSchemaObj["type"] = FieldTypeEnum.OBJECT.value + defaultCheckboxSchemaObj["properties"] = {} + # iterates over supported sources and sets the field for that source if field is present inside that source + for sourceType in dbConfig["supportedSourceTypes"]: + if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + defaultCheckboxSchemaObj["properties"][sourceType] = { + "type": FieldTypeEnum.BOOLEAN.value} + else: + defaultCheckboxSchemaObj["type"] = FieldTypeEnum.BOOLEAN.value + if "default" in field: + defaultCheckboxSchemaObj["default"] = field["default"] + return defaultCheckboxSchemaObj def generate_schema_for_checkbox(field, dbConfig, schema_field_name): @@ -909,7 +912,7 @@ def generate_config_props(config): generate_config_props(config) -def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): +def generate_schema(uiConfig, dbConfig, name, selector): """Returns the schema generated from given uiConfig and dbConfig. Args: @@ -917,7 +920,6 @@ def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): dbConfig (object): Configurations of db-config.json. name (string): name of the source or destination. selector (string): either 'source' or 'destination' - shouldUpdateSchema (boolean): if it should update the existing schema with generated one Returns: object: schema @@ -944,18 +946,6 @@ def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): schemaObject['properties'], name, selector) newSchema['configSchema'] = schemaObject - if shouldUpdateSchema: - # Get the parent directory (one level up) - script_directory = os.path.dirname(os.path.abspath(__file__)) - directory = os.path.dirname(script_directory) - # Define the relative path - relative_path = f'src/configurations/{selector}s/{name.lower()}/schema.json' - file_path = os.path.join(directory, relative_path) - new_content = json.dumps(newSchema) - # Write the new content - with open(file_path, 'w') as file: - file.write(new_content) - return newSchema def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): @@ -980,10 +970,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): curSchemaField = schema["properties"][field["value"]] newSchemaField = uiTypetoSchemaFn.get( curUiType)(field, dbConfig, "value") - schemaDiff = diff(newSchemaField, curSchemaField) + schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["value"], schemaDiff), UserWarning) + curUiType, field["value"], get_formatted_json(schemaDiff)), UserWarning) else: baseTemplate = uiConfig.get('baseTemplate', []) sdkTemplate = uiConfig.get('sdkTemplate', {}) @@ -1006,10 +996,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): curSchemaField = schema["properties"][field["configKey"]] newSchemaField = uiTypetoSchemaFn.get( curUiType)(field, dbConfig, "configKey") - schemaDiff = diff(newSchemaField, curSchemaField) + schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], schemaDiff), UserWarning) + curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) for field in sdkTemplate.get('fields', []): if "preRequisites" in field: @@ -1024,10 +1014,10 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): curSchemaField = schema["properties"][field["configKey"]] newSchemaField = uiTypetoSchemaFn.get( curUiType)(field, dbConfig, "configKey") - schemaDiff = diff(newSchemaField, curSchemaField) + schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], schemaDiff), UserWarning) + curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) for field in consentSettingsTemplate.get('fields', []): if "preRequisites" in field: @@ -1063,6 +1053,18 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): 'timePicker': generate_schema_for_time_picker } +def save_schema_to_file(selector, name, schema): + # Get the parent directory (one level up) + script_directory = os.path.dirname(os.path.abspath(__file__)) + directory = os.path.dirname(script_directory) + + # Define the relative path + relative_path = f'{CONFIG_DIR}/{selector}s/{name}/schema.json' + file_path = os.path.join(directory, relative_path) + + # Write the new content + with open(file_path, 'w') as file: + file.write(get_formatted_json(schema)) def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema): """Generates a schema and compares it with an existing one. @@ -1080,12 +1082,18 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou return if uiConfig == None: print('-'*50) - warnings.warn(f"Ui-Config is null for {name} in {selector} \n",UserWarning) + warnings.warn(f"Ui-Config is null for {name} in {selector} \n", UserWarning) print('-'*50) return - generatedSchema = generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema) + generatedSchema = generate_schema(uiConfig, dbConfig, name, selector) + if schema: - schemaDiff = diff(schema, generatedSchema["configSchema"]) + schemaDiff = get_json_diff(schema, generatedSchema["configSchema"]) + if shouldUpdateSchema: + finalSchema = {} + finalSchema["configSchema"] = apply_json_diff(schema, schemaDiff) + save_schema_to_file(selector, name, finalSchema) + if schemaDiff: print('-'*50) print(f'Schema diff for {name} in {selector}s') @@ -1101,30 +1109,33 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou else: curRequiredField = schema["required"] newRequiredField = generatedSchema["configSchema"]["required"] - requiredFieldDiff = diff(curRequiredField, newRequiredField) + requiredFieldDiff = get_json_diff(curRequiredField, newRequiredField) if requiredFieldDiff: - warnings.warn("For required field Difference is : \n\n {} \n".format(requiredFieldDiff), UserWarning) + warnings.warn("For required field Difference is : \n\n {} \n".format(get_formatted_json(requiredFieldDiff)), UserWarning) if "allOf" in generatedSchema["configSchema"]: curAllOfSchema = {} if "allOf" in schema: curAllOfSchema = schema["allOf"] newAllOfSchema = generatedSchema["configSchema"]["allOf"] - allOfSchemaDiff = diff(newAllOfSchema, curAllOfSchema) + allOfSchemaDiff = get_json_diff(curAllOfSchema, newAllOfSchema) if allOfSchemaDiff: - warnings.warn("For allOf field Difference is : \n\n {} \n".format(allOfSchemaDiff), UserWarning) + warnings.warn("For allOf field Difference is : \n\n {} \n".format(get_formatted_json(allOfSchemaDiff)), UserWarning) if "anyOf" in generatedSchema["configSchema"]: curAnyOfSchema = {} if "anyOf" in schema: curAnyOfSchema = schema["anyOf"] newAnyOfSchema = generatedSchema["configSchema"]["anyOf"] - anyOfSchemaDiff = diff(newAnyOfSchema, curAnyOfSchema) + anyOfSchemaDiff = get_json_diff(curAnyOfSchema, newAnyOfSchema) if anyOfSchemaDiff: - warnings.warn("For anyOf field Difference is : \n\n {} \n".format(anyOfSchemaDiff), UserWarning) + warnings.warn("For anyOf field Difference is : \n\n {} \n".format(get_formatted_json(anyOfSchemaDiff)), UserWarning) print('-'*50) else: + if shouldUpdateSchema: + save_schema_to_file(selector, name, generatedSchema) + print('-'*50) - print(f'Generated Schema for {name} in {selector}s') - print(json.dumps(generatedSchema,indent=2)) + print(f'Generated schema for {name} in {selector}s') + print(get_formatted_json(generatedSchema)) print('-'*50) def get_schema_diff(name, selector, shouldUpdateSchema=False): @@ -1135,20 +1146,24 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): selector (string): either 'source' or 'destination'. shouldUpdateSchema (boolean): if it should update the existing schema with generated one """ + file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] directory = f'./{CONFIG_DIR}/{selector}s/{name}' - available_files = os.listdir(directory) - file_content = {} - for file_selector in file_selectors: - if file_selector in available_files: - with open (f'{directory}/{file_selector}', 'r') as f: - file_content.update(json.loads(f.read())) - uiConfig = file_content.get("uiConfig") - schema = file_content.get("configSchema") - dbConfig = file_content.get("config") + if not os.path.isdir(directory): + print(f'No {selector}s directory found for {name}') + return + if name not in EXCLUDED_DEST: - validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) + available_files = os.listdir(directory) + file_content = {} + for file_selector in file_selectors: + if file_selector in available_files: + file_content.update(get_json_from_file(f'{directory}/{file_selector}')) + uiConfig = file_content.get("uiConfig") + schema = file_content.get("configSchema") + dbConfig = file_content.get("config") + validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json') @@ -1163,11 +1178,15 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): shouldUpdateSchema = args.update if args.all: - CONFIG_DIR = 'src/configurations' - current_items = os.listdir(f'./{CONFIG_DIR}/{selector}s') + dir_path = f'./{CONFIG_DIR}/{selector}s' + if not os.path.isdir(dir_path): + print(f'No {selector}s folder found') + exit(1) + + current_items = os.listdir(dir_path) for name in current_items: - get_schema_diff(name, selector) + get_schema_diff(name, selector, shouldUpdateSchema) else: - name = args.name + name = args.name get_schema_diff(name, selector, shouldUpdateSchema) diff --git a/scripts/template-db-config.json b/scripts/template-db-config.json index 0e8f50817..4205cc383 100644 --- a/scripts/template-db-config.json +++ b/scripts/template-db-config.json @@ -2,7 +2,6 @@ "name": "", "displayName": "", "config": { - "transformAt": "processor", "transformAtV1": "processor", "saveDestinationResponse": false, "includeKeys": [], @@ -14,12 +13,13 @@ "unity", "amp", "cloud", + "warehouse", "reactnative", "flutter", "cordova", - "warehouse" + "shopify" ], - "supportedConnectionModes": [], + "supportedConnectionModes": {}, "destConfig": { "defaultConfig": [] }, diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 000000000..49cc83027 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,51 @@ +from jsondiff import JsonDiffer +import json + +def get_json_diff(oldJson, newJson): + """Returns the difference between two JSONs. + + Args: + oldJson (object): old json. + newJson (object): new json. + + Returns: + object: difference between oldJson and newJson. + """ + differ = JsonDiffer(marshal=True) + return differ.diff(oldJson, newJson) + +def apply_json_diff(oldJson, diff): + """Applies the difference on oldJson and returns the newJson. + + Args: + oldJson (object): old json. + diff (object): difference between oldJson and newJson. + + Returns: + object: new json. + """ + differ = JsonDiffer(marshal=True) + return differ.patch(oldJson, diff) + +def get_formatted_json(jsonObj): + """Formats the json object. + + Args: + jsonObj (object): json object. + + Returns: + string: formatted json. + """ + return json.dumps(jsonObj, indent=2, ensure_ascii=False) + +def get_json_from_file(filePath): + """Reads the content of the file and returns the json object. + + Args: + filePath (string): file path. + + Returns: + object: json object. + """ + with open(filePath, 'r') as file: + return json.loads(file.read().encode('utf-8', 'ignore')) diff --git a/src/configurations/destinations/am/schema.json b/src/configurations/destinations/am/schema.json index 48782987f..fe02ce807 100644 --- a/src/configurations/destinations/am/schema.json +++ b/src/configurations/destinations/am/schema.json @@ -18,9 +18,9 @@ "properties": { "web": { "type": "string", - "pattern": "^(?!http:\/\/)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$" + "pattern": "^(?!http://)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$" } - } + } }, "trackAllPages": { "type": "boolean", diff --git a/src/configurations/destinations/am/ui-config.json b/src/configurations/destinations/am/ui-config.json index 12eba8b71..94524d683 100644 --- a/src/configurations/destinations/am/ui-config.json +++ b/src/configurations/destinations/am/ui-config.json @@ -543,7 +543,7 @@ "label": "Proxy server url", "note": "Send data to Amplitude by using a domain proxy to relay requests. Presently supported for web device mode only", "configKey": "proxyServerUrl", - "regex":"^(?!http:\/\/)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$", + "regex": "^(?!http://)(?:(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(?!.*\\.ngrok\\.io).*)$", "regexErrorMessage": "Invalid Proxy Server URL", "placeholder": "e.g: https://proxyserver.url.com" }, @@ -625,4 +625,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/configurations/destinations/spotifyPixel/db-config.json b/src/configurations/destinations/spotifyPixel/db-config.json index 95c3a933e..73801045e 100644 --- a/src/configurations/destinations/spotifyPixel/db-config.json +++ b/src/configurations/destinations/spotifyPixel/db-config.json @@ -1,41 +1,40 @@ { - "name": "SPOTIFYPIXEL", - "displayName": "Spotify Pixel", - "config": { - "transformAtV1": "processor", - "saveDestinationResponse": true, - "includeKeys": [ + "name": "SPOTIFYPIXEL", + "displayName": "Spotify Pixel", + "config": { + "transformAtV1": "processor", + "saveDestinationResponse": true, + "includeKeys": [ + "pixelId", + "eventsToSpotifyPixelEvents", + "blacklistedEvents", + "whitelistedEvents", + "oneTrustCookieCategories", + "eventFilteringOption", + "enableAliasCall" + ], + "excludeKeys": [], + "supportedSourceTypes": ["web"], + "supportedMessageTypes": { + "device": { + "web": ["track", "page"] + } + }, + "supportedConnectionModes": { + "web": ["device"] + }, + "destConfig": { + "defaultConfig": [ "pixelId", "eventsToSpotifyPixelEvents", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", - "enableAliasCall" + "enableAliasCall", + "oneTrustCookieCategories" ], - "excludeKeys": [], - "supportedSourceTypes": ["web"], - "supportedMessageTypes": { - "device": { - "web": ["track", "page"] - } - }, - "supportedConnectionModes": { - "web": ["device"] - }, - "destConfig": { - "defaultConfig": [ - "pixelId", - "eventsToSpotifyPixelEvents", - "blacklistedEvents", - "whitelistedEvents", - "eventFilteringOption", - "enableAliasCall", - "oneTrustCookieCategories" - ], - "web": ["useNativeSDK", "connectionMode"] - }, - "secretKeys": ["pixelId"] - } + "web": ["useNativeSDK", "connectionMode"] + }, + "secretKeys": ["pixelId"] } - \ No newline at end of file +} diff --git a/src/configurations/destinations/spotifyPixel/schema.json b/src/configurations/destinations/spotifyPixel/schema.json index 88588c530..ae51c344f 100644 --- a/src/configurations/destinations/spotifyPixel/schema.json +++ b/src/configurations/destinations/spotifyPixel/schema.json @@ -1,92 +1,91 @@ { - "configSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "required": ["pixelId"], - "type": "object", - "properties": { - "pixelId": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" - }, - "eventsToSpotifyPixelEvents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - }, - "to": { - "type": "string", - "enum": ["lead", "product", "addtocart", "checkout", "purchase", ""] - } - } - } - }, - "enableAliasCall": { - "type": "boolean", - "default": false - }, - "eventFilteringOption": { - "type": "string", - "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], - "default": "disable" - }, - "whitelistedEvents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "eventName": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["pixelId"], + "type": "object", + "properties": { + "pixelId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "eventsToSpotifyPixelEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "enum": ["lead", "product", "addtocart", "checkout", "purchase", ""] } } - }, - "blacklistedEvents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "eventName": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } + } + }, + "enableAliasCall": { + "type": "boolean", + "default": false + }, + "eventFilteringOption": { + "type": "string", + "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], + "default": "disable" + }, + "whitelistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" } } - }, - "useNativeSDK": { + } + }, + "blacklistedEvents": { + "type": "array", + "items": { "type": "object", "properties": { - "web": { - "type": "boolean" + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" } } - }, - "oneTrustCookieCategories": { - "type": "array", - "items": { - "type": "object", - "properties": { - "oneTrustCookieCategory": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } - } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" } - }, - "connectionMode": { + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { "type": "object", "properties": { - "web": { + "oneTrustCookieCategory": { "type": "string", - "enum": ["device"] + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { + "type": "string", + "enum": ["device"] + } + } } } } - \ No newline at end of file +} diff --git a/src/configurations/destinations/spotifyPixel/ui-config.json b/src/configurations/destinations/spotifyPixel/ui-config.json index 52e0c66b8..2ba754299 100644 --- a/src/configurations/destinations/spotifyPixel/ui-config.json +++ b/src/configurations/destinations/spotifyPixel/ui-config.json @@ -1,264 +1,264 @@ { - "uiConfig": { - "baseTemplate": [ - { - "title": "Initial setup", - "note": "Review how this destination is set up", - "sections": [ - { - "groups": [ - { - "title": "Connection Settings", - "note": "Update your connection settings here", - "icon": "settings", - "fields": [ - { - "type": "textInput", - "label": "Pixel ID", - "note": "Spotify Pixel dashboard > Manage > Your Pixels", - "configKey": "pixelId", - "regex": "^(.{1,100})$", - "secret": true, - "regexErrorMessage": "Invalid Pixel ID", - "placeholder": "e.g. dzq1p89h2bnpXXXX9x65hyx2hf5q1k3v" - } - ] - } - ] - }, - { - "groups": [ - { - "title": "Connection mode", - "note": [ - "Update how you want to route events from your source to destination. ", - { - "text": "Get help deciding", - "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" - } - ], - "icon": "sliders", - "fields": [], - "defaultConnectionModes": { - "web": "device" - } - } - ] - } + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Pixel ID", + "note": "Spotify Pixel dashboard > Manage > Your Pixels", + "configKey": "pixelId", + "regex": "^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Pixel ID", + "placeholder": "e.g. dzq1p89h2bnpXXXX9x65hyx2hf5q1k3v" + } ] - }, - { - "title": "Configuration settings", - "note": "Manage the settings for your destination", - "sections": [ + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [], + "defaultConnectionModes": { + "web": "device" + } + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Spotify Pixel Alias Event Setting", + "fields": [ + { + "type": "checkbox", + "label": "Enhance attribution with externalIDs for conversion matching", + "configKey": "enableAliasCall", + "default": false + } + ] + } + ] + }, + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "Client-side event filtering", + "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", + "preRequisites": { + "fields": [ { - "title": "Destination settings", - "note": "Configure advanced destination-specific settings here", - "icon": "settings", - "groups": [ - { - "title": "Spotify Pixel Alias Event Setting", - "fields": [ - { - "type": "checkbox", - "label": "Enhance attribution with externalIDs for conversion matching", - "configKey": "enableAliasCall", - "default": false - } - ] - } - ] + "configKey": "connectionModes.webDevice", + "value": true }, { - "title": "Other settings", - "note": "Configure advanced RudderStack features here", - "icon": "otherSettings", - "groups": [ - { - "title": "Client-side event filtering", - "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", - "preRequisites": { - "fields": [ - { - "configKey": "connectionModes.webDevice", - "value": true - }, - { - "configKey": "connectionModes.mobileDevice", - "value": true - } - ], - "condition": "or" - }, - "fields": [ - { - "type": "singleSelect", - "label": "Choose if you want to turn on events filtering:", - "configKey": "eventFilteringOption", - "note": "You must select either allowlist or denylist to enable events filtering", - "options": [ - { - "label": "No events filtering", - "value": "disable" - }, - { - "label": "Filter via allowlist", - "value": "whitelistedEvents" - }, - { - "label": "Filter via denylist", - "value": "blacklistedEvents" - } - ], - "default": "disable" - }, - { - "type": "tagInput", - "label": "Allowlisted events", - "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", - "configKey": "whitelistedEvents", - "tagKey": "eventName", - "placeholder": "e.g: Anonymous page visit", - "default": [ - { - "eventName": "" - } - ], - "preRequisites": { - "fields": [ - { - "configKey": "eventFilteringOption", - "value": "whitelistedEvents" - } - ] - } - }, - { - "type": "tagInput", - "label": "Denylisted events", - "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", - "configKey": "blacklistedEvents", - "tagKey": "eventName", - "placeholder": "e.g: Anonymous page visit", - "default": [ - { - "eventName": "" - } - ], - "preRequisites": { - "fields": [ - { - "configKey": "eventFilteringOption", - "value": "blacklistedEvents" - } - ] - } - } - ] - }, - { - "title": "OneTrust cookie consent settings", - "note": [ - "Enter your OneTrust category names if you have them configured. ", - { - "text": "Learn more ", - "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" - }, - "about RudderStack’s OneTrust Consent Manager feature." - ], - "fields": [ - { - "type": "tagInput", - "label": "Cookie category name", - "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", - "configKey": "oneTrustCookieCategories", - "tagKey": "oneTrustCookieCategory", - "placeholder": "e.g: Credit card visit", - "default": [ - { - "oneTrustCookieCategory": "" - } - ] - } - ] - } - ] + "configKey": "connectionModes.mobileDevice", + "value": true } - ] - }, - { - "title": "Mappings", - "hideEditIcon": true, - "sections": [ - { - "groups": [ - { - "title": "Rudderstack to Spotify Pixel Event Mappings", - "fields": [ - { - "type": "redirect", - "redirectGroupKey": "spotifyPixelEventMapping", - "label": "Event and property mappings" - } - ] - } - ] + ], + "condition": "or" + }, + "fields": [ + { + "type": "singleSelect", + "label": "Choose if you want to turn on events filtering:", + "configKey": "eventFilteringOption", + "note": "You must select either allowlist or denylist to enable events filtering", + "options": [ + { + "label": "No events filtering", + "value": "disable" + }, + { + "label": "Filter via allowlist", + "value": "whitelistedEvents" + }, + { + "label": "Filter via denylist", + "value": "blacklistedEvents" + } + ], + "default": "disable" + }, + { + "type": "tagInput", + "label": "Allowlisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", + "configKey": "whitelistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "whitelistedEvents" + } + ] } - ] - } - ], - "redirectGroups": { - "spotifyPixelEventMapping":{ - "fields": [ - { - "type": "mapping", - "label": "Mapping to trigger the Rudderstack events with standard Spotify Pixel events", - "configKey": "eventsToSpotifyPixelEvents", - "default": [], - "columns": [ - { - "type": "textInput", - "key": "from", - "label": "Event Name", - "placeholder": "e.g: Order Completed" - }, - { - "type": "singleSelect", - "key": "to", - "label": "Spotify Pixel Standard Event", - "placeholder": "e.g: Checkout", - "options": [ - { - "name": "Lead", - "value": "lead" - }, - { - "name": "Product", - "value": "product" - }, - { - "name": "Add to Cart", - "value": "addtocart" - }, - { - "name": "Check out", - "value": "checkout" - }, - { - "name": "Purchase", - "value": "purchase" - } - ] - } - ] + }, + { + "type": "tagInput", + "label": "Denylisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", + "configKey": "blacklistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "blacklistedEvents" + } + ] } - ] - } - }, - "sdkTemplate": { - "title": "Web SDK settings", - "note": "not visible in the ui", - "fields": [] - } + } + ] + }, + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Mappings", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "Rudderstack to Spotify Pixel Event Mappings", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "spotifyPixelEventMapping", + "label": "Event and property mappings" + } + ] + } + ] + } + ] + } + ], + "redirectGroups": { + "spotifyPixelEventMapping": { + "fields": [ + { + "type": "mapping", + "label": "Mapping to trigger the Rudderstack events with standard Spotify Pixel events", + "configKey": "eventsToSpotifyPixelEvents", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "from", + "label": "Event Name", + "placeholder": "e.g: Order Completed" + }, + { + "type": "singleSelect", + "key": "to", + "label": "Spotify Pixel Standard Event", + "placeholder": "e.g: Checkout", + "options": [ + { + "name": "Lead", + "value": "lead" + }, + { + "name": "Product", + "value": "product" + }, + { + "name": "Add to Cart", + "value": "addtocart" + }, + { + "name": "Check out", + "value": "checkout" + }, + { + "name": "Purchase", + "value": "purchase" + } + ] + } + ] + } + ] + } + }, + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] } -} \ No newline at end of file + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json new file mode 100644 index 000000000..1563ae3e7 --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "CREATE_NEW_SCHEMA_DESTINATION", + "displayName": "Create New Schema Destination", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "checkboxField", + "tagInputField" + ], + "web": ["webSourceField", "useNativeSDK", "connectionMode"], + "android": ["androidSourceField", "useNativeSDK", "connectionMode"] + }, + "secretKeys": ["secretTextInputField"] + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json new file mode 100644 index 000000000..86836f70b --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest/ui-config.json @@ -0,0 +1,118 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Test input field", + "configKey": "secretTextInputField", + "regex": "^(.{1,100})$", + "required": true, + "placeholder": "e.g. asdf123", + "secret": true + }, + { + "type": "singleSelect", + "label": "Single select field", + "configKey": "singleSelectField", + "options": [ + { + "label": "Value 1", + "value": "valOne" + }, + { + "label": "Value 2", + "value": "valTwo" + } + ], + "default": "valTwo" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Configure settings", + "fields": [ + { + "type": "checkbox", + "label": "A sample checkbox", + "configKey": "checkboxField", + "default": true + } + ] + }, + { + "type": "tagInput", + "label": "A tag input", + "configKey": "tagInputField", + "tagKey": "tagKey", + "placeholder": "e.g: abc", + "default": [] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [ + { + "type": "textInput", + "label": "A web source type field", + "configKey": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "configKey": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + } + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json new file mode 100644 index 000000000..560c62076 --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/db-config.json @@ -0,0 +1,49 @@ +{ + "name": "CREATE_NEW_SCHEMA_DESTINATION_LEGACY_UI", + "displayName": "Create New Schema Destination with Legacy UI", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "textareaInputField", + "checkboxField", + "dynamicCustomFormField", + "dynamicFormField", + "dynamicSelectFormField", + "timeRangePickerField", + "timePickerField" + ], + "web": ["webSourceField", "useNativeSDK"], + "android": ["androidSourceField", "useNativeSDK"] + }, + "secretKeys": ["secretTextInputField", "textareaInput"] + } +} diff --git a/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json new file mode 100644 index 000000000..751becd40 --- /dev/null +++ b/test/component_tests/configurations/destinations/create_new_schema_dest_legacy_ui/ui-config.json @@ -0,0 +1,153 @@ +{ + "uiConfig": [ + { + "title": "Connection Settings", + "fields": [ + { + "type": "textInput", + "label": "Access Token", + "value": "secretTextInputField", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "required": true, + "placeholder": "e.g: 96d96af0cdb847XXXXa4e7cc13255705", + "secret": true + }, + { + "type": "textInput", + "label": "A web source type field", + "value": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "value": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + }, + { + "title": "Native SDK", + "fields": [ + { + "type": "defaultCheckbox", + "label": "Use device-mode to send events", + "value": "useNativeSDK", + "default": true + } + ] + }, + { + "title": "Other Settings", + "fields": [ + { + "type": "textareaInput", + "subType": "JSON", + "label": "Textarea Input Field", + "value": "textareaInputField", + "regex": ".*", + "required": true, + "secret": true + }, + { + "type": "singleSelect", + "value": "singleSelectField", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ], + "defaultOption": { + "name": "Val 2", + "value": "valTwo" + } + }, + { + "type": "checkbox", + "label": "Checkbox field", + "value": "checkboxField", + "default": true + }, + { + "type": "dynamicCustomForm", + "value": "dynamicCustomFormField", + "label": "Dynamic custom form field", + "customFields": [ + { + "type": "textInput", + "value": "formField", + "required": false + } + ] + }, + { + "type": "dynamicForm", + "label": "Dynamic form field", + "labelRight": "Label right", + "labelLeft": "Label left", + "keyLeft": "left", + "keyRight": "right", + "value": "dynamicFormField" + }, + { + "type": "dynamicSelectForm", + "label": "Dynamic select form field", + "labelLeft": "Label left", + "labelRight": "Label right", + "value": "dynamicSelectFormField", + "keyLeft": "left", + "keyRight": "right", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ] + }, + { + "type": "timeRangePicker", + "label": "Time Range Picker Field", + "value": "timeRangePickerField", + "startTime": { + "label": "start time", + "value": "startTime" + }, + "endTime": { + "label": "end time", + "value": "endTime" + }, + "options": { + "omitSeconds": true, + "minuteStep": 1 + }, + "required": false + }, + { + "type": "timePicker", + "label": "Time Picker Field", + "value": "timePickerField", + "options": { + "omitSeconds": true, + "minuteStep": 15 + }, + "required": false + } + ] + } + ] +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest/db-config.json b/test/component_tests/configurations/destinations/update_schema_dest/db-config.json new file mode 100644 index 000000000..e2dee2339 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "UPDATE_SCHEMA_DESTINATION", + "displayName": "Update Schema Destination", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "checkboxField", + "tagInputField" + ], + "web": ["webSourceField", "useNativeSDK", "connectionMode"], + "android": ["androidSourceField", "useNativeSDK", "connectionMode"] + }, + "secretKeys": ["secretTextInputField"] + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest/schema.json b/test/component_tests/configurations/destinations/update_schema_dest/schema.json new file mode 100644 index 000000000..aea615f1b --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest/schema.json @@ -0,0 +1,92 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud", "device"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud", "device"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest/ui-config.json b/test/component_tests/configurations/destinations/update_schema_dest/ui-config.json new file mode 100644 index 000000000..86836f70b --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest/ui-config.json @@ -0,0 +1,118 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Test input field", + "configKey": "secretTextInputField", + "regex": "^(.{1,100})$", + "required": true, + "placeholder": "e.g. asdf123", + "secret": true + }, + { + "type": "singleSelect", + "label": "Single select field", + "configKey": "singleSelectField", + "options": [ + { + "label": "Value 1", + "value": "valOne" + }, + { + "label": "Value 2", + "value": "valTwo" + } + ], + "default": "valTwo" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [ + { + "title": "Configure settings", + "fields": [ + { + "type": "checkbox", + "label": "A sample checkbox", + "configKey": "checkboxField", + "default": true + } + ] + }, + { + "type": "tagInput", + "label": "A tag input", + "configKey": "tagInputField", + "tagKey": "tagKey", + "placeholder": "e.g: abc", + "default": [] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [ + { + "type": "textInput", + "label": "A web source type field", + "configKey": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "configKey": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + } + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json new file mode 100644 index 000000000..45833ffe8 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/db-config.json @@ -0,0 +1,49 @@ +{ + "name": "UPDATE_SCHEMA_DESTINATION_LEGACY_UI", + "displayName": "Update Schema Destination with Legacy UI", + "config": { + "transformAtV1": "processor", + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedConnectionModes": { + "android": ["cloud", "device"], + "ios": ["cloud"], + "web": ["cloud", "device"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "secretTextInputField", + "singleSelectField", + "textareaInputField", + "checkboxField", + "dynamicCustomFormField", + "dynamicFormField", + "dynamicSelectFormField", + "timeRangePickerField", + "timePickerField" + ], + "web": ["webSourceField", "useNativeSDK"], + "android": ["androidSourceField", "useNativeSDK"] + }, + "secretKeys": ["secretTextInputField", "textareaInput"] + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json new file mode 100644 index 000000000..9790d9ff6 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/schema.json @@ -0,0 +1,106 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" + } + } + }, + "textareaInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "dynamicCustomFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "formField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicSelectFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "timeRangePickerField": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + } + }, + "required": ["startTime", "endTime"] + }, + "timePickerField": { + "type": "string" + } + } + } +} diff --git a/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json new file mode 100644 index 000000000..751becd40 --- /dev/null +++ b/test/component_tests/configurations/destinations/update_schema_dest_legacy_ui/ui-config.json @@ -0,0 +1,153 @@ +{ + "uiConfig": [ + { + "title": "Connection Settings", + "fields": [ + { + "type": "textInput", + "label": "Access Token", + "value": "secretTextInputField", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "required": true, + "placeholder": "e.g: 96d96af0cdb847XXXXa4e7cc13255705", + "secret": true + }, + { + "type": "textInput", + "label": "A web source type field", + "value": "webSourceField", + "default": "1000", + "regex": "^([0-9]{0,100})$", + "placeholder": "e.g: 1000" + }, + { + "type": "textInput", + "label": "An android source type field", + "value": "androidSourceField", + "regex": "^([0-9]{0,100})$", + "default": "30", + "placeholder": "e.g: 30" + } + ] + }, + { + "title": "Native SDK", + "fields": [ + { + "type": "defaultCheckbox", + "label": "Use device-mode to send events", + "value": "useNativeSDK", + "default": true + } + ] + }, + { + "title": "Other Settings", + "fields": [ + { + "type": "textareaInput", + "subType": "JSON", + "label": "Textarea Input Field", + "value": "textareaInputField", + "regex": ".*", + "required": true, + "secret": true + }, + { + "type": "singleSelect", + "value": "singleSelectField", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ], + "defaultOption": { + "name": "Val 2", + "value": "valTwo" + } + }, + { + "type": "checkbox", + "label": "Checkbox field", + "value": "checkboxField", + "default": true + }, + { + "type": "dynamicCustomForm", + "value": "dynamicCustomFormField", + "label": "Dynamic custom form field", + "customFields": [ + { + "type": "textInput", + "value": "formField", + "required": false + } + ] + }, + { + "type": "dynamicForm", + "label": "Dynamic form field", + "labelRight": "Label right", + "labelLeft": "Label left", + "keyLeft": "left", + "keyRight": "right", + "value": "dynamicFormField" + }, + { + "type": "dynamicSelectForm", + "label": "Dynamic select form field", + "labelLeft": "Label left", + "labelRight": "Label right", + "value": "dynamicSelectFormField", + "keyLeft": "left", + "keyRight": "right", + "required": false, + "options": [ + { + "name": "Val 1", + "value": "valOne" + }, + { + "name": "Val 2", + "value": "valTwo" + } + ] + }, + { + "type": "timeRangePicker", + "label": "Time Range Picker Field", + "value": "timeRangePickerField", + "startTime": { + "label": "start time", + "value": "startTime" + }, + "endTime": { + "label": "end time", + "value": "endTime" + }, + "options": { + "omitSeconds": true, + "minuteStep": 1 + }, + "required": false + }, + { + "type": "timePicker", + "label": "Time Picker Field", + "value": "timePickerField", + "options": { + "omitSeconds": true, + "minuteStep": 15 + }, + "required": false + } + ] + } + ] +} diff --git a/test/component_tests/data/createNewSchemaDest.json b/test/component_tests/data/createNewSchemaDest.json new file mode 100644 index 000000000..97fb27e5c --- /dev/null +++ b/test/component_tests/data/createNewSchemaDest.json @@ -0,0 +1,100 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "singleSelectField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud", "device"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud", "device"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/test/component_tests/data/createNewSchemaDestLegacyUI.json b/test/component_tests/data/createNewSchemaDestLegacyUI.json new file mode 100644 index 000000000..ddf0c78c4 --- /dev/null +++ b/test/component_tests/data/createNewSchemaDestLegacyUI.json @@ -0,0 +1,114 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "textareaInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "textareaInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "dynamicCustomFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "formField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicSelectFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "timeRangePickerField": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + } + }, + "required": ["startTime", "endTime"] + }, + "timePickerField": { + "type": "string" + } + } + } +} diff --git a/test/component_tests/data/updateSchemaDest.json b/test/component_tests/data/updateSchemaDest.json new file mode 100644 index 000000000..97fb27e5c --- /dev/null +++ b/test/component_tests/data/updateSchemaDest.json @@ -0,0 +1,100 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "singleSelectField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud", "device"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud", "device"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/test/component_tests/data/updateSchemaDestLegacyUI.json b/test/component_tests/data/updateSchemaDestLegacyUI.json new file mode 100644 index 000000000..ddf0c78c4 --- /dev/null +++ b/test/component_tests/data/updateSchemaDestLegacyUI.json @@ -0,0 +1,114 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["secretTextInputField", "textareaInputField"], + "type": "object", + "properties": { + "secretTextInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "webSourceField": { + "type": "object", + "properties": { + "web": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "androidSourceField": { + "type": "object", + "properties": { + "android": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]{0,100})$" + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "android": { + "type": "boolean" + }, + "web": { + "type": "boolean" + } + } + }, + "textareaInputField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*" + }, + "singleSelectField": { + "type": "string", + "enum": ["valOne", "valTwo"], + "default": "valTwo" + }, + "checkboxField": { + "type": "boolean", + "default": true + }, + "dynamicCustomFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "formField": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "dynamicSelectFormField": { + "type": "array", + "items": { + "type": "object", + "properties": { + "left": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "right": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "timeRangePickerField": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + } + }, + "required": ["startTime", "endTime"] + }, + "timePickerField": { + "type": "string" + } + } + } +} diff --git a/test/component_tests/schemaGenerator.test.ts b/test/component_tests/schemaGenerator.test.ts new file mode 100644 index 000000000..54fb1f3b9 --- /dev/null +++ b/test/component_tests/schemaGenerator.test.ts @@ -0,0 +1,151 @@ +import fs from 'fs'; +import path from 'path'; +import { execSync } from 'child_process'; + +function readSchemaFile(filePath): string | undefined { + let schema; + if (!fs.existsSync(filePath)) { + return schema; + } + + try { + schema = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch (e) { + /* empty */ + } + + return schema; +} + +function writeSchemaFile(filePath, schema) { + fs.writeFileSync(filePath, JSON.stringify(schema, null, 2)); +} + +describe('Schema Generator', () => { + const configDir = 'test/component_tests/configurations'; + + describe('should generate and save schema for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'create_new_schema_dest', + expectedSchemaFile: 'createNewSchemaDest.json', + }, + { + description: 'Legacy UI', + destName: 'create_new_schema_dest_legacy_ui', + expectedSchemaFile: 'createNewSchemaDestLegacyUI.json', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName, expectedSchemaFile }) => { + const cmd = `CONFIG_DIR=${configDir} npm run update:schema:destination "${destName}"`; + + execSync(cmd); + + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + + const schema = readSchemaFile(schemaFilePath); + if (schema) { + // delete the file + fs.unlinkSync(schemaFilePath); + } + + const expectedSchemaData = readSchemaFile( + path.resolve(__dirname, `./data/${expectedSchemaFile}`), + ); + + expect(schema).toEqual(expectedSchemaData); + }); + }); + + describe('should not generate and save schema if update option is not provided for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'create_new_schema_dest', + }, + { + description: 'Legacy UI', + destName: 'create_new_schema_dest_legacy_ui', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName }) => { + const cmd = `CONFIG_DIR=${configDir} npm run check:schema:destination "${destName}"`; + + execSync(cmd); + + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + + const schema = readSchemaFile(schemaFilePath); + expect(schema).toBeUndefined(); + }); + }); + + describe('should update and save schema for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'update_schema_dest', + expectedSchemaFile: 'updateSchemaDest.json', + }, + { + description: 'Legacy UI', + destName: 'update_schema_dest_legacy_ui', + expectedSchemaFile: 'updateSchemaDestLegacyUI.json', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName, expectedSchemaFile }) => { + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + const curSchema = readSchemaFile(schemaFilePath); + + const cmd = `CONFIG_DIR=${configDir} npm run update:schema:destination "${destName}"`; + + execSync(cmd); + + const schema = readSchemaFile(schemaFilePath); + // Restore schema file + if (schema) { + writeSchemaFile(schemaFilePath, curSchema); + } + + const expectedSchemaData = readSchemaFile( + path.resolve(__dirname, `./data/${expectedSchemaFile}`), + ); + + expect(schema).toEqual(expectedSchemaData); + }); + }); + + describe('should not save the schema file if update option is not provided for the specified destination', () => { + const testData = [ + { + description: 'New UI', + destName: 'update_schema_dest', + }, + { + description: 'Legacy UI', + destName: 'update_schema_dest_legacy_ui', + }, + ]; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.each(testData)('$description', ({ description, destName }) => { + const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); + const curSchema = readSchemaFile(schemaFilePath); + + const cmd = `CONFIG_DIR=${configDir} npm run check:schema:destination "${destName}"`; + + execSync(cmd); + + const schema = readSchemaFile(schemaFilePath); + + expect(schema).toEqual(curSchema); + }); + }); +}); diff --git a/test/data/validation/destinations/spotifyPixel.json b/test/data/validation/destinations/spotifyPixel.json index bd932cacc..d349ef3e6 100644 --- a/test/data/validation/destinations/spotifyPixel.json +++ b/test/data/validation/destinations/spotifyPixel.json @@ -27,9 +27,7 @@ } }, "result": false, - "err": [ - "pixelId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" - ] + "err": ["pixelId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] }, { "config": { @@ -59,9 +57,7 @@ } }, "result": false, - "err": [ - " must have required property 'pixelId'" - ] + "err": [" must have required property 'pixelId'"] }, { "config": { @@ -92,9 +88,7 @@ } }, "result": false, - "err": [ - "pixelId must be string" - ] + "err": ["pixelId must be string"] }, { "config": { @@ -153,8 +147,6 @@ } }, "result": false, - "err": [ - "eventsToSpotifyPixelEvents.0.to must be equal to one of the allowed values" - ] + "err": ["eventsToSpotifyPixelEvents.0.to must be equal to one of the allowed values"] } -] \ No newline at end of file +] From fdbb379e81ed16b29303b0e0b2d8cf6771ea8a56 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:20:43 +0530 Subject: [PATCH 02/20] chore: setup python before running tests (#1199) --- .github/workflows/deploy.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bea52c72b..06cb58655 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,22 +43,22 @@ jobs: run: | npm ci - - name: Execute Unit Tests - env: - HUSKY: 0 - run: | - npm run test:ci - - name: Install Python uses: actions/setup-python@v5.0.0 with: python-version: '3.9.13' - + - name: Display Python Version run: | which python python -c "import sys; print(sys.version)" + - name: Execute Unit Tests + env: + HUSKY: 0 + run: | + npm run test:ci + - name: Install Python Dependencies run: pip3 install -r ./scripts/requirements.txt From 6edc55b1db32139945365defbd4b0e462ce33339 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:40:34 +0530 Subject: [PATCH 03/20] chore: use proper python setup step (#1200) --- .github/workflows/deploy.yml | 8 +++----- .github/workflows/report-code-coverage.yml | 2 +- .../destinations/tiktok_ads/db-config.json | 13 ++++++++++++- test/component_tests/schemaGenerator.test.ts | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 06cb58655..24e2d6ab2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,11 +43,9 @@ jobs: run: | npm ci - - name: Install Python - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.9.13' - + - name: Set up Python + run: scripts/setup-python.sh + - name: Display Python Version run: | which python diff --git a/.github/workflows/report-code-coverage.yml b/.github/workflows/report-code-coverage.yml index 9791eb5d7..c65c16c76 100644 --- a/.github/workflows/report-code-coverage.yml +++ b/.github/workflows/report-code-coverage.yml @@ -26,7 +26,7 @@ jobs: - name: Install Dependencies run: npm ci - + - name: Set up Python run: scripts/setup-python.sh diff --git a/src/configurations/destinations/tiktok_ads/db-config.json b/src/configurations/destinations/tiktok_ads/db-config.json index b56a92240..ed7c780fe 100644 --- a/src/configurations/destinations/tiktok_ads/db-config.json +++ b/src/configurations/destinations/tiktok_ads/db-config.json @@ -17,7 +17,18 @@ "ketchConsentPurposes" ], "excludeKeys": [], - "supportedSourceTypes": ["web", "cloud", "ios", "android", "unity", "amp", "warehouse", "reactnative", "flutter", "cordova"], + "supportedSourceTypes": [ + "web", + "cloud", + "ios", + "android", + "unity", + "amp", + "warehouse", + "reactnative", + "flutter", + "cordova" + ], "supportedMessageTypes": { "cloud": ["track"], "device": { diff --git a/test/component_tests/schemaGenerator.test.ts b/test/component_tests/schemaGenerator.test.ts index 54fb1f3b9..91a2f546e 100644 --- a/test/component_tests/schemaGenerator.test.ts +++ b/test/component_tests/schemaGenerator.test.ts @@ -110,7 +110,7 @@ describe('Schema Generator', () => { const schema = readSchemaFile(schemaFilePath); // Restore schema file - if (schema) { + if (curSchema) { writeSchemaFile(schemaFilePath, curSchema); } From 009c8da1f0089f8413f105c2443a233cd597148a Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Mon, 12 Feb 2024 10:36:14 +0530 Subject: [PATCH 04/20] chore: standard formatting (#1198) --- .github/workflows/commitlint.yml | 36 ++ .github/workflows/verify.yml | 53 ++ .husky/pre-commit | 2 +- migration/migrate.py | 108 ++-- package.json | 5 +- scripts/configGenerator.py | 48 +- scripts/constants.py | 10 +- scripts/deployToDB.py | 111 ++-- scripts/schemaGenerator.py | 869 ++++++++++++++++++++----------- scripts/setup-python.sh | 1 + scripts/utils.py | 16 +- test/test_configGenerator.py | 14 +- 12 files changed, 837 insertions(+), 436 deletions(-) create mode 100644 .github/workflows/commitlint.yml create mode 100644 .github/workflows/verify.yml diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml new file mode 100644 index 000000000..a8ff39eee --- /dev/null +++ b/.github/workflows/commitlint.yml @@ -0,0 +1,36 @@ +name: Commitlint + +on: [push] + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + + - name: Setup Node + uses: actions/setup-node@v4.0.1 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install Dependencies + run: npm ci + + - name: Print versions + run: | + git --version + node --version + npm --version + npx commitlint --version + + # Run the commitlint action, considering its own dependencies and yours as well 🚀 + # `github.workspace` is the path to your repository. + - uses: wagoid/commitlint-github-action@v5 + env: + NODE_PATH: ${{ github.workspace }}/node_modules + with: + commitDepth: 1 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 000000000..90ec08d77 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,53 @@ +name: Code quality checks + +on: + pull_request: + +jobs: + py-scripts-lint: + name: Check for formatting of python scripts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Setting up python libraries + run: ./scripts/setup-python.sh + + # Reference: https://black.readthedocs.io/en/stable/integrations/github_actions.html + - name: Check formatting for Python files + uses: psf/black@stable + with: + options: '--check --verbose' + + formatting-lint: + name: Check for formatting & lint errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - name: Setup Node + uses: actions/setup-node@v3.7.0 + with: + node-version-file: .nvmrc + cache: 'npm' + + - name: Install Dependencies + run: npm ci + + - name: Run Lint Checks + run: | + npm run lint + + - run: git diff --exit-code + + - name: Error message + if: ${{ failure() }} + run: | + echo 'ESLint check is failing. Please run `npm run lint` on your working copy and commit the changes.' diff --git a/.husky/pre-commit b/.husky/pre-commit index d4a43dd13..14ccce4a2 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,4 @@ #!/usr/bin/env sh . "$(dirname -- "$0")/_/husky.sh" -npm run pre-commit +npm run pre-commit \ No newline at end of file diff --git a/migration/migrate.py b/migration/migrate.py index 79c958304..6106730e2 100644 --- a/migration/migrate.py +++ b/migration/migrate.py @@ -2,116 +2,126 @@ import json import os -DEST_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/destinations/schemas/' -DEST_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/destinationConfigs/' -DEST_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/' +DEST_WEBAPP_FILE_PATH = "rudder-webapp/src/components/destinations/schemas/" +DEST_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/destinationConfigs/" +DEST_SCHEMA_FILE_PATH = "rudder-config-backend/src/scripts/schemaList/destinations/" -SOURCE_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/' -SOURCE_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/sourceConfigs/' +SOURCE_WEBAPP_FILE_PATH = ( + "rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/" +) +SOURCE_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/sourceConfigs/" # SOURCE_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/' -CONFIGURATIONS_DIR_PATH = '../src/configurations' -DEST_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/destinations' -SRC_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/sources' +CONFIGURATIONS_DIR_PATH = "../src/configurations" +DEST_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/destinations" +SRC_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/sources" + def update_destination(): - dest_list = [d[:-5].lower() for d in os.listdir(f'../../{DEST_CONFIG_BE_FILE_PATH}')] + dest_list = [ + d[:-5].lower() for d in os.listdir(f"../../{DEST_CONFIG_BE_FILE_PATH}") + ] for dest in dest_list: final_data = [] # read db_config - with open(f'../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json', 'r') as f: + with open(f"../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json", "r") as f: db_config = json.loads(f.read()) - print (db_config) + print(db_config) final_data.append(db_config) - print ("========================") + print("========================") # read ui_config - with open(f'../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json', 'r') as f: + with open(f"../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json", "r") as f: ui_config = {"uiConfig": json.loads(f.read())} - print (ui_config) + print(ui_config) final_data.append(ui_config) - print ("========================") + print("========================") # read schema - if f'{dest.upper()}.json' in os.listdir(f'../../{DEST_SCHEMA_FILE_PATH}'): - with open(f'../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json', 'r') as f: + if f"{dest.upper()}.json" in os.listdir(f"../../{DEST_SCHEMA_FILE_PATH}"): + with open(f"../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json", "r") as f: schema = {"schema": json.loads(f.read())} else: schema = {"schema": None} - print (schema) + print(schema) final_data.append(schema) - print ("========================") + print("========================") # metadata - with open('dest_meta_template.json', 'r') as f: + with open("dest_meta_template.json", "r") as f: metadata = json.loads(f.read()) - print (metadata) + print(metadata) final_data.append(metadata) - print ("========================") + print("========================") ######################## ## write new files ######################## if dest not in os.listdir(DEST_CONFIG_PATH): - os.system(f'mkdir {DEST_CONFIG_PATH}/{dest}') - print (f'created directory for {dest}') + os.system(f"mkdir {DEST_CONFIG_PATH}/{dest}") + print(f"created directory for {dest}") - file_names = ['db-config', 'ui-config', 'schema', 'metadata'] + file_names = ["db-config", "ui-config", "schema", "metadata"] for index, f_name in enumerate(file_names): - print (f'writing {f_name} for {dest}...') - with open(f'{DEST_CONFIG_PATH}/{dest}/{f_name}.json', 'w') as f: + print(f"writing {f_name} for {dest}...") + with open(f"{DEST_CONFIG_PATH}/{dest}/{f_name}.json", "w") as f: f.write(json.dumps(final_data[index], indent=2)) - print (f'complete for {dest}...') - print ('------------------------------------------------') + print(f"complete for {dest}...") + print("------------------------------------------------") + def update_source(): - source_list = [d[:-5].lower() for d in os.listdir(f'../../{SOURCE_CONFIG_BE_FILE_PATH}')] + source_list = [ + d[:-5].lower() for d in os.listdir(f"../../{SOURCE_CONFIG_BE_FILE_PATH}") + ] for source in source_list: final_data = [] # read db_config - with open(f'../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json', 'r') as f: + with open(f"../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json", "r") as f: db_config = json.loads(f.read()) - print (db_config) + print(db_config) final_data.append(db_config) - print ("========================") + print("========================") # read ui_config - if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_WEBAPP_FILE_PATH}'): - with open(f'../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json', 'r') as f: + if f"{source.upper()}.json" in os.listdir(f"../../{SOURCE_WEBAPP_FILE_PATH}"): + with open( + f"../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json", "r" + ) as f: ui_config = {"uiConfig": json.loads(f.read())} else: ui_config = {"uiConfig": None} - print (ui_config) + print(ui_config) final_data.append(ui_config) - print ("========================") + print("========================") # read schema (we don't need schema for sources) # if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_SCHEMA_FILE_PATH}'): # with open(f'../../{SOURCE_SCHEMA_FILE_PATH}{source.upper()}.json', 'r') as f: # schema = {"schema": json.loads(f.read())} # else: schema = {"schema": None} - print (schema) + print(schema) final_data.append(schema) - print ("========================") + print("========================") # metadata - with open('source_meta_template.json', 'r') as f: + with open("source_meta_template.json", "r") as f: metadata = json.loads(f.read()) - print (metadata) + print(metadata) final_data.append(metadata) - print ("========================") + print("========================") ######################## ## write new files ######################## if source not in os.listdir(SRC_CONFIG_PATH): - os.system(f'mkdir {SRC_CONFIG_PATH}/{source}') - print (f'created directory for {source}') + os.system(f"mkdir {SRC_CONFIG_PATH}/{source}") + print(f"created directory for {source}") - file_names = ['db_config', 'ui_config', 'schema', 'metadata'] + file_names = ["db_config", "ui_config", "schema", "metadata"] for index, f_name in enumerate(file_names): - print (f'writing {f_name} for {source}...') - with open(f'{SRC_CONFIG_PATH}/{source}/{f_name}.json', 'w') as f: + print(f"writing {f_name} for {source}...") + with open(f"{SRC_CONFIG_PATH}/{source}/{f_name}.json", "w") as f: f.write(json.dumps(final_data[index], indent=2)) - print (f'complete for {source}...') - print ('------------------------------------------------') + print(f"complete for {source}...") + print("------------------------------------------------") if __name__ == "__main__": diff --git a/package.json b/package.json index 534f118ac..167e25fd0 100755 --- a/package.json +++ b/package.json @@ -21,8 +21,10 @@ "test:silent": "npm run test -- --silent", "release": "npx standard-version", "check:lint": "eslint \"src/**/*.*\" -f json -o reports/eslint.json || exit 0", + "format:py": "python3 -m black .", "format": "prettier . --write", "lint:fix": "eslint \"src/**/*.*\" --fix", + "lint": "npm run format && npm run lint:fix", "prepare": "husky install", "pre-commit": "npm run test && npx lint-staged", "commit-msg": "commitlint --edit", @@ -79,7 +81,8 @@ "glob": "^9.3.2" }, "lint-staged": { - "*.{json,js,ts,md}": "prettier --write" + "*.{json,js,ts,md}": "prettier --write", + "*.{py}": "python3 -m black" }, "config": { "commitizen": { diff --git a/scripts/configGenerator.py b/scripts/configGenerator.py index 1a1fdaf4d..f7e38ba0c 100644 --- a/scripts/configGenerator.py +++ b/scripts/configGenerator.py @@ -4,23 +4,24 @@ import os import sys -ConfigData = TypedDict('ConfigData', {'db_config': str, 'ui_config': str}) +ConfigData = TypedDict("ConfigData", {"db_config": str, "ui_config": str}) + def generateConfigs(data) -> ConfigData: # Read the content of template-db-config.json - with open('scripts/template-db-config.json', 'r') as file: + with open("scripts/template-db-config.json", "r") as file: template_db_config = json.load(file) # Read the content of template-ui-config.json - with open('scripts/template-ui-config.json', 'r') as file: + with open("scripts/template-ui-config.json", "r") as file: template_ui_config = json.load(file) # Create db-config object with the same content db_config = template_db_config - db_config['displayName'] = data['displayName'] - db_config['name'] = data['displayName'] - formFields = data['formFields'] + db_config["displayName"] = data["displayName"] + db_config["name"] = data["displayName"] + formFields = data["formFields"] # Create db-config object with the same content ui_config = template_ui_config @@ -33,14 +34,13 @@ def appendFieldsInGroups(settings, field): if groups: first_group = groups[0] if "fields" in first_group: - del field['required'] + del field["required"] first_group["fields"].append(field) - def updateUiConfig(field): if "uiConfig" in ui_config and "baseTemplate" in ui_config["uiConfig"]: base_template = ui_config["uiConfig"]["baseTemplate"] - if base_template and field['required'] == True: + if base_template and field["required"] == True: connection_settings = base_template[0] appendFieldsInGroups(connection_settings, field) elif base_template: @@ -48,28 +48,27 @@ def updateUiConfig(field): appendFieldsInGroups(configuration_settings, field) return ui_config - # Iterate over JSON objects in the array for obj in formFields: # update field in ui-config ui_config = updateUiConfig(obj) # update db-config for key, value in obj.items(): - if key == 'configKey': - db_config['config']['destConfig']['defaultConfig'].append( - value) - if key == 'secret' and value == True: - db_config['config']['secretKeys'].append( - db_config['config']['destConfig']['defaultConfig'][-1]) + if key == "configKey": + db_config["config"]["destConfig"]["defaultConfig"].append(value) + if key == "secret" and value == True: + db_config["config"]["secretKeys"].append( + db_config["config"]["destConfig"]["defaultConfig"][-1] + ) db_config = json.dumps(db_config) ui_config = json.dumps(ui_config) - return {'db_config':db_config, 'ui_config': ui_config} + return {"db_config": db_config, "ui_config": ui_config} -if __name__ == '__main__': - file_path = sys.argv[1] if len(sys.argv) > 1 else 'test/configData/inputData.json' - with open(file_path, 'r') as file: +if __name__ == "__main__": + file_path = sys.argv[1] if len(sys.argv) > 1 else "test/configData/inputData.json" + with open(file_path, "r") as file: # Load the JSON data data = json.load(file) @@ -79,16 +78,15 @@ def updateUiConfig(field): if not os.path.exists(directory): os.makedirs(directory) - with open(file_path, 'w') as file: + with open(file_path, "w") as file: # Write the new content - file.write(configData['db_config']) - + file.write(configData["db_config"]) file_path = f'src/configurations/destinations/{data["displayName"]}/ui-config.json' directory = os.path.dirname(file_path) if not os.path.exists(directory): os.makedirs(directory) - with open(file_path, 'w') as file: + with open(file_path, "w") as file: # Write the new content - file.write(configData['ui_config']) + file.write(configData["ui_config"]) diff --git a/scripts/constants.py b/scripts/constants.py index 54b717202..5d9c7b926 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -1,11 +1,11 @@ import os -__DEFAULT_CONFIG_DIR = 'src/configurations' +__DEFAULT_CONFIG_DIR = "src/configurations" CONFIG_DIR = __DEFAULT_CONFIG_DIR -if 'CONFIG_DIR' in os.environ: - CONFIG_DIR = os.environ['CONFIG_DIR'] +if "CONFIG_DIR" in os.environ: + CONFIG_DIR = os.environ["CONFIG_DIR"] -TEST_INTEGRATION_NAME_PREFIX = 'test_' +TEST_INTEGRATION_NAME_PREFIX = "test_" -TEST_INTEGRATION_NAME_SUFFIX = '_ignore' +TEST_INTEGRATION_NAME_SUFFIX = "_ignore" diff --git a/scripts/deployToDB.py b/scripts/deployToDB.py index 6c513fb92..97b4b3d00 100644 --- a/scripts/deployToDB.py +++ b/scripts/deployToDB.py @@ -18,12 +18,12 @@ ######################### # ENV VARIABLES -CONTROL_PLANE_URL=sys.argv[1] +CONTROL_PLANE_URL = sys.argv[1] print(CONTROL_PLANE_URL) -USERNAME=os.environ['API_USER'] #sys.argv[2] +USERNAME = os.environ["API_USER"] # sys.argv[2] print(USERNAME) -PASSWORD=os.environ['API_PASSWORD'] #sys.argv[3] -#print(PASSWORD) +PASSWORD = os.environ["API_PASSWORD"] # sys.argv[3] +# print(PASSWORD) ######################### # CONSTANTS HEADER = {"Content-Type": "application/json"} @@ -34,131 +34,152 @@ ######################### # UTIL METHODS def parse_response(resp): - if resp.status_code >= 200 and resp.status_code <= 300: - return resp.status_code, resp.json() - else: - return resp.status_code, str(resp.content) + if resp.status_code >= 200 and resp.status_code <= 300: + return resp.status_code, resp.json() + else: + return resp.status_code, str(resp.content) + def get_persisted_store(base_url, selector): - request_url = f'{base_url}/{selector}-definitions' + request_url = f"{base_url}/{selector}-definitions" response = requests.get(request_url) return json.loads(response.text) + def get_config_definition(base_url, selector, name): - request_url = f'{base_url}/{selector}-definitions/{name}' + request_url = f"{base_url}/{selector}-definitions/{name}" response = requests.get(request_url) return response + def get_file_content(name, selector): - file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] + file_selectors = ["db-config.json", "ui-config.json", "schema.json"] - directory = f'./{CONFIG_DIR}/{selector}s/{name}' + directory = f"./{CONFIG_DIR}/{selector}s/{name}" available_files = os.listdir(directory) file_content = {} for file_selector in file_selectors: if file_selector in available_files: - with open (f'{directory}/{file_selector}', 'r') as f: + with open(f"{directory}/{file_selector}", "r") as f: file_content.update(json.loads(f.read())) return file_content + def update_config_definition(selector, name, fileData): - url = f'{CONTROL_PLANE_URL}/{selector}-definitions/{name}' + url = f"{CONTROL_PLANE_URL}/{selector}-definitions/{name}" resp = requests.post(url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH) return parse_response(resp) + def create_config_definition(selector, fileData): - url = f'{CONTROL_PLANE_URL}/{selector}-definitions/' + url = f"{CONTROL_PLANE_URL}/{selector}-definitions/" resp = requests.post(url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH) return parse_response(resp) + def update_config(data_diff, selector): results = [] for diff in data_diff: - name = diff['name'] + name = diff["name"] fileData = get_file_content(name, selector) nameInConfig = fileData["name"] - if diff['action'] == 'create': - url = f'{CONTROL_PLANE_URL}/{selector}-definitions' + if diff["action"] == "create": + url = f"{CONTROL_PLANE_URL}/{selector}-definitions" else: - url = f'{CONTROL_PLANE_URL}/{selector}-definitions/{nameInConfig}' + url = f"{CONTROL_PLANE_URL}/{selector}-definitions/{nameInConfig}" - resp = requests.post(url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH) + resp = requests.post( + url=url, headers=HEADER, data=json.dumps(fileData), auth=AUTH + ) status, response = parse_response(resp) - diff['update'] = {"status": status, "response": response} + diff["update"] = {"status": status, "response": response} # results.append(diff) results.append(name) - return json.dumps(results, indent=2) + def update_diff_db(selector): final_report = [] ## data sets - current_items = os.listdir(f'./{CONFIG_DIR}/{selector}s') + current_items = os.listdir(f"./{CONFIG_DIR}/{selector}s") for item in current_items: # check if item is a directory - if not os.path.isdir(f'./{CONFIG_DIR}/{selector}s/{item}'): + if not os.path.isdir(f"./{CONFIG_DIR}/{selector}s/{item}"): continue updated_data = get_file_content(item, selector) - persisted_data = get_config_definition(CONTROL_PLANE_URL, selector, updated_data["name"]) + persisted_data = get_config_definition( + CONTROL_PLANE_URL, selector, updated_data["name"] + ) if persisted_data.status_code == 200: - diff = jsondiff.diff(json.loads(persisted_data.text), updated_data, marshal=True) + diff = jsondiff.diff( + json.loads(persisted_data.text), updated_data, marshal=True + ) # ignore the $delete - values present in DB but missing in files. Anyways this doesn't get reflected in DB as keys are missing in files itself. # Best practice is to make sure all keys are maintained in the config files irrespective of them being null. - del diff['$delete'] - - if len(diff.keys()) > 0: # changes exist - #print(diff) - status, response = update_config_definition(selector, updated_data["name"], updated_data) - final_report.append({"name": updated_data["name"], "action":"update", "status": status}) + del diff["$delete"] + + if len(diff.keys()) > 0: # changes exist + # print(diff) + status, response = update_config_definition( + selector, updated_data["name"], updated_data + ) + final_report.append( + {"name": updated_data["name"], "action": "update", "status": status} + ) else: - final_report.append({"name": updated_data["name"], "action":"na", "status": ""}) + final_report.append( + {"name": updated_data["name"], "action": "na", "status": ""} + ) else: status, response = create_config_definition(selector, updated_data) - final_report.append({"name": updated_data["name"], "action":"create", "status": status}) + final_report.append( + {"name": updated_data["name"], "action": "create", "status": status} + ) return final_report + def get_stale_data(selector, report): stale_config_report = [] persisted_data_set = get_persisted_store(CONTROL_PLANE_URL, selector) - persisted_items = [item['name'] for item in persisted_data_set] - file_items = [item['name'] for item in report] + persisted_items = [item["name"] for item in persisted_data_set] + file_items = [item["name"] for item in report] for item in persisted_items: if item not in file_items: - stale_config_report.append({item}) + stale_config_report.append({item}) return stale_config_report -if __name__ == '__main__': + +if __name__ == "__main__": print("Running Destination Definitions Updates") - dest_final_report = update_diff_db('destination') + dest_final_report = update_diff_db("destination") print("Destination Definition Update Report") print(dest_final_report) print("Destination Stale Config Report") - print(get_stale_data('destination', dest_final_report)) + print(get_stale_data("destination", dest_final_report)) print("Running Source Definitions Updates") - src_final_report = update_diff_db('source') + src_final_report = update_diff_db("source") print("Source Definition Update Report") print(src_final_report) print("Source Stale Config Report") - print(get_stale_data('source', src_final_report)) + print(get_stale_data("source", src_final_report)) print("Running Wht Lib Projects Definitions Updates") - wht_final_report = update_diff_db('wht-lib-project') + wht_final_report = update_diff_db("wht-lib-project") print("Wht lib project Definition Update Report") print(wht_final_report) print("Wht lib project Stale Config Report") - print(get_stale_data('wht-lib-project', wht_final_report)) - \ No newline at end of file + print(get_stale_data("wht-lib-project", wht_final_report)) diff --git a/scripts/schemaGenerator.py b/scripts/schemaGenerator.py index 36d9d9ee3..60f97a0cf 100644 --- a/scripts/schemaGenerator.py +++ b/scripts/schemaGenerator.py @@ -1,4 +1,4 @@ -''' +""" Usage: schemaGenerator.py [-h] [-name name | -all] [-update] selector 1. selector - “source” or “destination” 2. all - runs the validator for all the selector. @@ -7,7 +7,8 @@ Example: 1. python3 scripts/schemaGenerator.py -name="adobe_analytics" destination 2. python3 scripts/schemaGenerator.py -all source -''' +""" + import os import warnings from enum import Enum @@ -15,7 +16,8 @@ from utils import get_json_from_file, get_json_diff, apply_json_diff, get_formatted_json from constants import CONFIG_DIR -EXCLUDED_DEST = ['postgres', 'bq', 'azure_synapse', 'clickhouse', 'deltalake', 'kafka'] +EXCLUDED_DEST = ["postgres", "bq", "azure_synapse", "clickhouse", "deltalake", "kafka"] + class FieldTypeEnum(Enum): STRING = "string" @@ -29,13 +31,14 @@ def is_old_format(uiConfig): return False return True + def get_options_list_for_enum(field): """Creates the list of options given in field and return the list Args: field (object): Individual field in ui-config. Returns: list: list of options - """ + """ options_list = [] for i in range(0, len(field["options"])): if isinstance(field["options"][i], int) or isinstance(field["options"][i], str): @@ -43,14 +46,19 @@ def get_options_list_for_enum(field): else: options_list.append(field["options"][i]["value"]) # allow empty field in enum if field in not required. - if "default" not in field and "defaultOption" not in field and field.get("required", False) == False: - options_list.append("") + if ( + "default" not in field + and "defaultOption" not in field + and field.get("required", False) == False + ): + options_list.append("") return options_list + def generalize_regex_pattern(field): """Generates the pattern for schema based on the type of field. - For type : singleSelect and dynamicSelectForm, the pattern is generated by iterating over options. - - For other types, + - For other types, - If the field contains regex, then regex is the pattern; it gets prefixed with a default prefix if regex does not have it. - Else, the default prefix is appended with ^(.{0,100}). @@ -59,88 +67,116 @@ def generalize_regex_pattern(field): Returns: string: generated pattern for the field. - """ + """ defaultSubPattern = "(^\\{\\{.*\\|\\|(.*)\\}\\}$)" defaultEnvPattern = "(^env[.].+)" pattern = "" if "regex" in field: pattern = field["regex"] - if defaultSubPattern not in pattern and (('value' not in field or field['value'] != 'purpose') and ('configKey' not in field or field['configKey'] != 'purpose')): + if defaultSubPattern not in pattern and ( + ("value" not in field or field["value"] != "purpose") + and ("configKey" not in field or field["configKey"] != "purpose") + ): pattern = "|".join([defaultSubPattern, pattern]) - if defaultEnvPattern not in pattern and (('value' not in field or field['value'] != 'purpose') and ('configKey' not in field or field['configKey'] != 'purpose')): + if defaultEnvPattern not in pattern and ( + ("value" not in field or field["value"] != "purpose") + and ("configKey" not in field or field["configKey"] != "purpose") + ): indexToPlace = pattern.find(defaultSubPattern) + len(defaultSubPattern) - pattern = pattern[:indexToPlace] + '|' + defaultEnvPattern + pattern[indexToPlace:] + pattern = ( + pattern[:indexToPlace] + + "|" + + defaultEnvPattern + + pattern[indexToPlace:] + ) # TODO: we should not use a case here for the individual properties. Just pass the desired pattern as regex property # in ketch purpose fields and delete next case - elif ('value' in field and field['value'] == 'purpose') or ('configKey' in field and field['configKey'] == 'purpose'): - pattern = '^(.{0,100})$' + elif ("value" in field and field["value"] == "purpose") or ( + "configKey" in field and field["configKey"] == "purpose" + ): + pattern = "^(.{0,100})$" else: - pattern = "|".join([defaultSubPattern, defaultEnvPattern, '^(.{0,100})$']) + pattern = "|".join([defaultSubPattern, defaultEnvPattern, "^(.{0,100})$"]) return pattern def is_dest_field_dependent_on_source(field, dbConfig, schema_field_name): - """Checks if the given field is source-specific by using dbConfig. - In dbConfig all the sources are listed in 'supportedSourceTypes', + """Checks if the given field is source-specific by using dbConfig. + In dbConfig all the sources are listed in 'supportedSourceTypes', and their fields are listed inside 'destConfig' with the key as the source. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: boolean: True if the field is source dependent else, False. - """ + """ if not dbConfig: return False for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): return True return False + def is_field_present_in_default_config(field, dbConfig, schema_field_name): """Checks if the given field is present in defaultConfig list present in dbConfig. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: boolean: True if field is in defaultConfig else False. - """ + """ if not dbConfig: return False - if "destConfig" in dbConfig and "defaultConfig" in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"]["defaultConfig"]: + if ( + "destConfig" in dbConfig + and "defaultConfig" in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"]["defaultConfig"] + ): return True return False + def generate_schema_for_default_checkbox(field, dbConfig, schema_field_name): """Creates a schema object of defaultCheckbox. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object - """ - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + """ + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) defaultCheckboxSchemaObj = {} if isSourceDependent: defaultCheckboxSchemaObj["type"] = FieldTypeEnum.OBJECT.value defaultCheckboxSchemaObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): defaultCheckboxSchemaObj["properties"][sourceType] = { - "type": FieldTypeEnum.BOOLEAN.value} + "type": FieldTypeEnum.BOOLEAN.value + } else: defaultCheckboxSchemaObj["type"] = FieldTypeEnum.BOOLEAN.value if "default" in field: @@ -154,22 +190,28 @@ def generate_schema_for_checkbox(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) checkboxSchemaObj = {} if isSourceDependent: checkboxSchemaObj["type"] = FieldTypeEnum.OBJECT.value checkboxSchemaObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): checkboxSchemaObj["properties"][sourceType] = { - "type": FieldTypeEnum.BOOLEAN.value} + "type": FieldTypeEnum.BOOLEAN.value + } else: checkboxSchemaObj["type"] = FieldTypeEnum.BOOLEAN.value if "default" in field: @@ -183,27 +225,35 @@ def generate_schema_for_textinput(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ textInputSchemaObj = {} - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) if isSourceDependent: textInputSchemaObj["type"] = FieldTypeEnum.OBJECT.value textInputSchemaObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): textInputSchemaObj["properties"][sourceType] = { - "type": FieldTypeEnum.STRING.value} - if 'regex' in field: - textInputSchemaObj["properties"][sourceType]["pattern"] = generalize_regex_pattern(field) + "type": FieldTypeEnum.STRING.value + } + if "regex" in field: + textInputSchemaObj["properties"][sourceType]["pattern"] = ( + generalize_regex_pattern(field) + ) else: textInputSchemaObj = {"type": FieldTypeEnum.STRING.value} - if 'regex' in field: + if "regex" in field: textInputSchemaObj["pattern"] = generalize_regex_pattern(field) return textInputSchemaObj @@ -214,14 +264,14 @@ def generate_schema_for_textarea_input(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ textareaInputObj = {"type": FieldTypeEnum.STRING.value} - if 'regex' in field: + if "regex" in field: textareaInputObj["pattern"] = generalize_regex_pattern(field) return textareaInputObj @@ -232,25 +282,25 @@ def generate_schema_for_single_select(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ singleSelectObj = {} - if "mode" in field and field["mode"] == 'multiple': - singleSelectObj = {"type": FieldTypeEnum.ARRAY.value} + if "mode" in field and field["mode"] == "multiple": + singleSelectObj = {"type": FieldTypeEnum.ARRAY.value} singleSelectObj["items"] = { "type": FieldTypeEnum.STRING.value, - "enum": get_options_list_for_enum(field) + "enum": get_options_list_for_enum(field), } if "default" or "defaultOption" in field: if isinstance(field["defaultOption"]["value"], list): singleSelectObj["default"] = field["defaultOption"]["value"] elif field["defaultOption"]["value"]: singleSelectObj["default"] = [field["defaultOption"]["value"]] - elif 'default' in field: + elif "default" in field: singleSelectObj["default"] = field["default"] else: singleSelectObj = {"type": FieldTypeEnum.STRING.value} @@ -258,16 +308,21 @@ def generate_schema_for_single_select(field, dbConfig, schema_field_name): if "default" or "defaultOption" in field: if "defaultOption" in field: singleSelectObj["default"] = field["defaultOption"]["value"] - elif 'default' in field: + elif "default" in field: singleSelectObj["default"] = field["default"] - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) if isSourceDependent: newSingleSelectObj = {"type": FieldTypeEnum.OBJECT.value} newSingleSelectObj["properties"] = {} # iterates over supported sources and sets the field for that source if field is present inside that source for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): newSingleSelectObj["properties"][sourceType] = singleSelectObj singleSelectObj = newSingleSelectObj return singleSelectObj @@ -279,7 +334,7 @@ def generate_schema_for_dynamic_custom_form(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -297,46 +352,71 @@ def generate_schema_for_dynamic_custom_form(field, dbConfig, schema_field_name): if "rowFields" in field: customFieldsKey = "rowFields" - allOfSchemaObj = generate_schema_for_dynamic_custom_form_allOf(field[customFieldsKey], dbConfig, schema_field_name) + allOfSchemaObj = generate_schema_for_dynamic_custom_form_allOf( + field[customFieldsKey], dbConfig, schema_field_name + ) for customField in field[customFieldsKey]: - customFieldSchemaObj = uiTypetoSchemaFn.get(customField["type"])(customField, dbConfig, schema_field_name) - isCustomFieldDependentOnSource = is_dest_field_dependent_on_source(customField, dbConfig, schema_field_name) + customFieldSchemaObj = uiTypetoSchemaFn.get(customField["type"])( + customField, dbConfig, schema_field_name + ) + isCustomFieldDependentOnSource = is_dest_field_dependent_on_source( + customField, dbConfig, schema_field_name + ) if "preRequisites" in customField: continue - if 'pattern' not in customFieldSchemaObj and not isCustomFieldDependentOnSource and customFieldSchemaObj["type"] == FieldTypeEnum.STRING.value and customField["type"] != "singleSelect" and customField["type"] != "dynamicSelectForm": + if ( + "pattern" not in customFieldSchemaObj + and not isCustomFieldDependentOnSource + and customFieldSchemaObj["type"] == FieldTypeEnum.STRING.value + and customField["type"] != "singleSelect" + and customField["type"] != "dynamicSelectForm" + ): customFieldSchemaObj["pattern"] = generalize_regex_pattern(customField) # If the custom field is source dependent, we remove the source keys as it's not required inside custom fields, rather they need to be moved to top. if isCustomFieldDependentOnSource: for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: - customFieldSchemaObj = customFieldSchemaObj["properties"][sourceType] + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): + customFieldSchemaObj = customFieldSchemaObj["properties"][ + sourceType + ] break - dynamicCustomFormItemObj["properties"][customField[schema_field_name]] = customFieldSchemaObj + dynamicCustomFormItemObj["properties"][ + customField[schema_field_name] + ] = customFieldSchemaObj if allOfSchemaObj: - dynamicCustomFormItemObj['allOf'] = allOfSchemaObj + dynamicCustomFormItemObj["allOf"] = allOfSchemaObj dynamicCustomFormObj["items"] = dynamicCustomFormItemObj - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) # If the field is source dependent, new schema object is created by setting the fields inside the source. if isSourceDependent: newDynamicCustomFormObj = {"type": FieldTypeEnum.OBJECT.value} newDynamicCustomFormObj["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): newDynamicCustomFormObj["properties"][sourceType] = dynamicCustomFormObj dynamicCustomFormObj = newDynamicCustomFormObj return dynamicCustomFormObj - -def generate_schema_for_dynamic_custom_form_allOf(customFields, dbConfig, schema_field_name): +def generate_schema_for_dynamic_custom_form_allOf( + customFields, dbConfig, schema_field_name +): """Creates the allOf structure of schema, empty if not required. - Finds the list of unique preRequisites. - For each unique preRequisites, the properties are found by matching the current preRequisites. @@ -359,7 +439,9 @@ def generate_schema_for_dynamic_custom_form_allOf(customFields, dbConfig, schema continue isPresent = False for preRequisites in preRequisitesList: - if compare_pre_requisite_fields(preRequisites, field["preRequisites"]["fields"], True): + if compare_pre_requisite_fields( + preRequisites, field["preRequisites"]["fields"], True + ): isPresent = True break if not isPresent: @@ -373,8 +455,12 @@ def generate_schema_for_dynamic_custom_form_allOf(customFields, dbConfig, schema for field in customFields: if "preRequisites" not in field: continue - if compare_pre_requisite_fields(field["preRequisites"]["fields"], preRequisites, True): - thenObj["properties"][field[schema_field_name]] = uiTypetoSchemaFn.get(field["type"])(field, dbConfig, schema_field_name) + if compare_pre_requisite_fields( + field["preRequisites"]["fields"], preRequisites, True + ): + thenObj["properties"][field[schema_field_name]] = uiTypetoSchemaFn.get( + field["type"] + )(field, dbConfig, schema_field_name) if "required" in field and field["required"] == True: thenObj["required"].append(field[schema_field_name]) allOfItemObj["then"] = thenObj @@ -391,18 +477,19 @@ def generate_schema_for_dynamic_form(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ + def generate_key(forFieldWithTo): obj = { "type": FieldTypeEnum.STRING.value, } - if(field["type"] == 'dynamicSelectForm'): - if (forFieldWithTo != (field.get("reverse", False)==False)): + if field["type"] == "dynamicSelectForm": + if forFieldWithTo != (field.get("reverse", False) == False): obj["pattern"] = generalize_regex_pattern(field) else: if "defaultOption" in field: @@ -413,25 +500,35 @@ def generate_key(forFieldWithTo): return obj dynamicFormSchemaObject = {} - dynamicFormSchemaObject['type'] = FieldTypeEnum.ARRAY.value + dynamicFormSchemaObject["type"] = FieldTypeEnum.ARRAY.value dynamicFormItemObject = {} dynamicFormItemObject["type"] = FieldTypeEnum.OBJECT.value - dynamicFormItemObject['properties'] = {} + dynamicFormItemObject["properties"] = {} dynamicFormItemObjectProps = [ - (field['keyLeft'], generate_key), (field['keyRight'], generate_key)] + (field["keyLeft"], generate_key), + (field["keyRight"], generate_key), + ] for dynamicFromItemObjectProp in dynamicFormItemObjectProps: - dynamicFormItemObject['properties'][dynamicFromItemObjectProp[0] - ] = dynamicFromItemObjectProp[1](dynamicFromItemObjectProp[0] == "to") - dynamicFormSchemaObject['items'] = dynamicFormItemObject - - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + dynamicFormItemObject["properties"][dynamicFromItemObjectProp[0]] = ( + dynamicFromItemObjectProp[1](dynamicFromItemObjectProp[0] == "to") + ) + dynamicFormSchemaObject["items"] = dynamicFormItemObject + + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) # If the field is source dependent, new schema object is created by setting the fields inside the source. if isSourceDependent: newDynamicFormFormObj = {"type": FieldTypeEnum.OBJECT.value} newDynamicFormFormObj["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: - newDynamicFormFormObj["properties"][sourceType] = dynamicFormSchemaObject + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): + newDynamicFormFormObj["properties"][ + sourceType + ] = dynamicFormSchemaObject dynamicFormSchemaObject = newDynamicFormFormObj return dynamicFormSchemaObject @@ -442,7 +539,7 @@ def generate_schema_for_dynamic_select_form(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -450,13 +547,14 @@ def generate_schema_for_dynamic_select_form(field, dbConfig, schema_field_name): """ return generate_schema_for_dynamic_form(field, dbConfig, schema_field_name) + def generate_schema_for_mapping(field, dbConfig, schema_field_name): """Creates a schema object of mapping. Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -471,7 +569,7 @@ def generate_schema_for_tag_input(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -480,23 +578,28 @@ def generate_schema_for_tag_input(field, dbConfig, schema_field_name): tagObject = {} tagObject["type"] = FieldTypeEnum.ARRAY.value tagItem = {} - tagItem['type'] = FieldTypeEnum.OBJECT.value + tagItem["type"] = FieldTypeEnum.OBJECT.value tagItemProps = { - str(field['tagKey']): { + str(field["tagKey"]): { "type": FieldTypeEnum.STRING.value, - "pattern": generalize_regex_pattern(field) + "pattern": generalize_regex_pattern(field), } } - tagItem['properties'] = tagItemProps + tagItem["properties"] = tagItemProps tagObject["items"] = tagItem - isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + isSourceDependent = is_dest_field_dependent_on_source( + field, dbConfig, schema_field_name + ) if isSourceDependent: tagObjectCopy = tagObject tagObject = {} tagObject = {"type": FieldTypeEnum.OBJECT.value} tagObject["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + if ( + sourceType in dbConfig["destConfig"] + and field[schema_field_name] in dbConfig["destConfig"][sourceType] + ): tagObject["properties"][sourceType] = tagObjectCopy return tagObject @@ -507,20 +610,20 @@ def generate_schema_for_time_range_picker(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ timeRangeObj = {} - timeRangeObj['type'] = FieldTypeEnum.OBJECT.value + timeRangeObj["type"] = FieldTypeEnum.OBJECT.value timeRangeProps = { - field['startTime']['value']: {'type': FieldTypeEnum.STRING.value}, - field['endTime']['value']: {'type': FieldTypeEnum.STRING.value} + field["startTime"]["value"]: {"type": FieldTypeEnum.STRING.value}, + field["endTime"]["value"]: {"type": FieldTypeEnum.STRING.value}, } - timeRangeObj['properties'] = timeRangeProps - timeRangeObj['required'] = list(timeRangeProps.keys()) + timeRangeObj["properties"] = timeRangeProps + timeRangeObj["required"] = list(timeRangeProps.keys()) return timeRangeObj @@ -530,18 +633,16 @@ def generate_schema_for_time_picker(field, dbConfig, schema_field_name): Args: field (object): Individual field in ui-config. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object """ - return { - "type": FieldTypeEnum.STRING.value - } + return {"type": FieldTypeEnum.STRING.value} -def compare_pre_requisite_fields(fieldA, fieldB, isV2 = False): +def compare_pre_requisite_fields(fieldA, fieldB, isV2=False): """Compares two preRequisiteFields fieldA and fieldB for each property and checks if their value matches. Args: @@ -552,12 +653,12 @@ def compare_pre_requisite_fields(fieldA, fieldB, isV2 = False): Returns: boolean: If all the properties have the same 'name' and 'selectedValue', then it returns True else False. """ - valueKey = 'selectedValue' - nameKey = 'name' + valueKey = "selectedValue" + nameKey = "name" if isV2: - valueKey = 'value' - nameKey = 'configKey' + valueKey = "value" + nameKey = "configKey" if type(fieldA) != type(fieldB): return False @@ -565,13 +666,17 @@ def compare_pre_requisite_fields(fieldA, fieldB, isV2 = False): if len(fieldA) != len(fieldB): return False for i in range(0, len(fieldA)): - if fieldA[i][nameKey] != fieldB[i][nameKey] or fieldA[i][valueKey] != fieldB[i][valueKey]: + if ( + fieldA[i][nameKey] != fieldB[i][nameKey] + or fieldA[i][valueKey] != fieldB[i][valueKey] + ): return False else: if fieldA[nameKey] != fieldB[nameKey] or fieldA[valueKey] != fieldB[valueKey]: - return False + return False return True + def get_unique_pre_requisite_fields(uiConfig): """Returns the list of unique preRequisiteFields present in a uiConfig. @@ -580,16 +685,18 @@ def get_unique_pre_requisite_fields(uiConfig): Returns: list: containing unique preRequisiteFields. - """ + """ preRequisiteFieldsList = [] for group in uiConfig: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: if "preRequisiteField" not in field: continue isPresent = False for preRequisiteField in preRequisiteFieldsList: - if compare_pre_requisite_fields(preRequisiteField, field["preRequisiteField"]): + if compare_pre_requisite_fields( + preRequisiteField, field["preRequisiteField"] + ): isPresent = True break if not isPresent: @@ -597,7 +704,7 @@ def get_unique_pre_requisite_fields(uiConfig): return preRequisiteFieldsList -def generate_if_object(preRequisiteField, isV2 = False): +def generate_if_object(preRequisiteField, isV2=False): """Creates an if object for the given preRequisiteField. The preRequisiteField becomes an if condition in the schema. Args: @@ -608,18 +715,16 @@ def generate_if_object(preRequisiteField, isV2 = False): object: if block for given preRequisiteField. """ ifObj = {"properties": {}, "required": []} - valueKey = 'selectedValue' - nameKey = 'name' + valueKey = "selectedValue" + nameKey = "name" if isV2: - valueKey = 'value' - nameKey = 'configKey' + valueKey = "value" + nameKey = "configKey" if type(preRequisiteField) == list: for field in preRequisiteField: - ifObj["properties"][field[nameKey]] = { - "const": field[valueKey] - } + ifObj["properties"][field[nameKey]] = {"const": field[valueKey]} ifObj["required"].append(field[nameKey]) else: ifObj["properties"][preRequisiteField[nameKey]] = { @@ -638,12 +743,12 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): Args: uiConfig (object): file content of ui-config.json. dbConfig (object): Configurations of db-config.json. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: object: allOf object of schema - """ + """ allOfItemList = [] preRequisiteFieldsList = get_unique_pre_requisite_fields(uiConfig) for preRequisiteField in preRequisiteFieldsList: @@ -651,12 +756,18 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): thenObj = {"properties": {}, "required": []} allOfItemObj = {"if": ifObj} for group in uiConfig: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: if "preRequisiteField" not in field: continue - if compare_pre_requisite_fields(field["preRequisiteField"], preRequisiteField): - thenObj["properties"][field[schema_field_name]] = uiTypetoSchemaFn.get(field["type"])(field, dbConfig, schema_field_name) + if compare_pre_requisite_fields( + field["preRequisiteField"], preRequisiteField + ): + thenObj["properties"][field[schema_field_name]] = ( + uiTypetoSchemaFn.get(field["type"])( + field, dbConfig, schema_field_name + ) + ) if "required" in field and field["required"] == True: thenObj["required"].append(field[schema_field_name]) allOfItemObj["then"] = thenObj @@ -665,14 +776,15 @@ def generate_schema_for_allOf(uiConfig, dbConfig, schema_field_name): allOfItemList = generate_schema_for_anyOf(allOfItemList, schema_field_name) return allOfItemList + def get_common_and_opposite_fields(propertiesA, propertiesB, schema_field_name): """Takes properties of two if Objects and returns a list of common and opposite properties. Common properties have the same value in both A and B. Args: propertiesA (object): if block - propertiesB (object): - schema_field_name (string): Specifies which key has the field's name in schema. + propertiesB (object): + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: @@ -684,10 +796,10 @@ def get_common_and_opposite_fields(propertiesA, propertiesB, schema_field_name): propertiesA = { 'storage' : {'const' : 'S3'}, 'useStorage' : {'const': True}} propertiesB = { 'storage' : {'const' : 'S3'}, 'useStorage' : {'const': False}} - Returns + Returns commonProperties = [{'key': 'storage', 'value':'S3'}] oppositeProperties = [{'key': 'useStorage', 'value': 'True'}] - """ + """ keysListA = list(propertiesA.keys()) keysListB = list(propertiesB.keys()) commonProperties = [] @@ -696,32 +808,40 @@ def get_common_and_opposite_fields(propertiesA, propertiesB, schema_field_name): if key not in keysListB: return None, None if propertiesA[key]["const"] == propertiesB[key]["const"]: - commonProperties.append({"key": key, schema_field_name: propertiesA[key]["const"]}) - elif type(propertiesA[key]["const"]) == bool and propertiesA[key]["const"] != propertiesB[key]["const"]: - oppositeProperties.append({"key": key, schema_field_name: propertiesA[key]["const"]}) + commonProperties.append( + {"key": key, schema_field_name: propertiesA[key]["const"]} + ) + elif ( + type(propertiesA[key]["const"]) == bool + and propertiesA[key]["const"] != propertiesB[key]["const"] + ): + oppositeProperties.append( + {"key": key, schema_field_name: propertiesA[key]["const"]} + ) else: return None, None return commonProperties, oppositeProperties + def check_if_conditions_match(ifPropsA, ifObjectB, schema_field_name): """Compares the ifPropsA and ifObjectB if they have the same properties and values. Args: ifPropsA (list): consists of key and "schema_field_name" pairs. ifObjectB (object): if block consisting of multiple properties with the value. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: boolean: True if all the properties in IfPropsA are in ifObjectB with same value, else False. - + Example: - schema_field_name = 'value' + schema_field_name = 'value' ifPropsA = [{'key': 'storage', 'value': 'S3'}, {'key': 'deploy', 'value': 'web'}] ifObjectA = {'storage': {'const': 'S3'}, 'deploy': {'const': 'web'}} - + Returns: True - """ + """ for ifProp in ifPropsA: if ifProp["key"] not in ifObjectB: return False @@ -729,18 +849,19 @@ def check_if_conditions_match(ifPropsA, ifObjectB, schema_field_name): return False return True + def find_index_to_place_anyOf(ifProp, allOfItemList, schema_field_name): """Returns the index of the item in allOfItemList consisting of matching if conditions as that of ifProp. Args: ifProp (object): consists of key and "schema_field_name" pairs. allOfItemList (list): consists of a list of objects with "if-then" properties. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: int: index of if-block matching with ifProp else -1. - + Example: schema_field_name = 'value' ifProp = [{'key': 'storage', 'value': 'S3'}, {'key': 'deploy', 'value': 'web'}] @@ -750,85 +871,109 @@ def find_index_to_place_anyOf(ifProp, allOfItemList, schema_field_name): ] Returns: 1 - """ + """ if not ifProp: return -1 length = len(allOfItemList) for index in range(length): - if "if" in allOfItemList[index] and check_if_conditions_match(ifProp, allOfItemList[index]["if"]["properties"], schema_field_name): + if "if" in allOfItemList[index] and check_if_conditions_match( + ifProp, allOfItemList[index]["if"]["properties"], schema_field_name + ): return index return -1 + def generate_schema_for_anyOf(allOfItemList, schema_field_name): """Takes in two parameters allOfItemList and schema_field_name, and returns an updated allOf Items list. - - It checks for all the pairs of allOf Items ("if-then" blocks), if their "if" conditions have an "if-else" based structure rather than "if-if". + - It checks for all the pairs of allOf Items ("if-then" blocks), if their "if" conditions have an "if-else" based structure rather than "if-if". - Items following the "if-else" structure are deleted and replaced by an anyOf structure. Args: allOfItemList (list): consists of a list of objects with "if-then" properties. - schema_field_name (string): Specifies which key has the field's name in schema. + schema_field_name (string): Specifies which key has the field's name in schema. For old schema types, it is 'value' else 'configKey'. Returns: list: updated allOf Items list - """ + """ length = len(allOfItemList) delIndices = [] for i in range(0, length): - for j in range(i+1, length): + for j in range(i + 1, length): ifPropertiesA = allOfItemList[i]["if"]["properties"] thenPropertiesA = allOfItemList[i]["then"] ifPropertiesB = allOfItemList[j]["if"]["properties"] thenPropertiesB = allOfItemList[j]["then"] - commonIfProp, oppositeIfProp = get_common_and_opposite_fields(ifPropertiesA, ifPropertiesB, schema_field_name) + commonIfProp, oppositeIfProp = get_common_and_opposite_fields( + ifPropertiesA, ifPropertiesB, schema_field_name + ) if oppositeIfProp: anyOfObj = [{}, {}] for k in range(0, len(oppositeIfProp)): if ifPropertiesA[oppositeIfProp[k]["key"]]["const"] == True: anyOfObj[1] = thenPropertiesA - anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = {"const": True} + anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = { + "const": True + } anyOfObj[1]["required"].append(oppositeIfProp[k]["key"]) anyOfObj[0] = thenPropertiesB - anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = {"const": False} + anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = { + "const": False + } else: anyOfObj[1] = thenPropertiesB - anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = {"const": True} + anyOfObj[1]["properties"][oppositeIfProp[k]["key"]] = { + "const": True + } anyOfObj[1]["required"].append(oppositeIfProp[k]["key"]) anyOfObj[0] = thenPropertiesA - anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = {"const": False} - # AnyOf object is placed at index of "if-then" block having same if properties as of common properties else at end. - indexToPlace = find_index_to_place_anyOf(commonIfProp, allOfItemList, schema_field_name) + anyOfObj[0]["properties"][oppositeIfProp[k]["key"]] = { + "const": False + } + # AnyOf object is placed at index of "if-then" block having same if properties as of common properties else at end. + indexToPlace = find_index_to_place_anyOf( + commonIfProp, allOfItemList, schema_field_name + ) if indexToPlace != -1: allOfItemList[indexToPlace]["then"]["anyOf"] = anyOfObj delIndices.append(i) delIndices.append(j) - allOfItemList = [allOfItemList[index] for index in range(len(allOfItemList)) if index not in delIndices] + allOfItemList = [ + allOfItemList[index] + for index in range(len(allOfItemList)) + if index not in delIndices + ] return allOfItemList + def generate_connection_mode(dbConfig): """Creates the connection mode object present in new schema types. - + Args: dbConfig (object): Configurations of db-config.json. Returns: object - """ + """ connectionObj = {"type": FieldTypeEnum.OBJECT.value} connectionObj["properties"] = {} for sourceType in dbConfig["supportedSourceTypes"]: - if sourceType in dbConfig["supportedConnectionModes"]: - connectionItemObj = {"type": FieldTypeEnum.STRING.value} - connectionModesEnum=[] - length = len(dbConfig["supportedConnectionModes"][sourceType]) - for i in range(0, length): - connectionModesEnum.append(dbConfig["supportedConnectionModes"][sourceType][i]) - connectionItemObj["enum"] = connectionModesEnum - connectionObj["properties"][sourceType] = connectionItemObj + if sourceType in dbConfig["supportedConnectionModes"]: + connectionItemObj = {"type": FieldTypeEnum.STRING.value} + connectionModesEnum = [] + length = len(dbConfig["supportedConnectionModes"][sourceType]) + for i in range(0, length): + connectionModesEnum.append( + dbConfig["supportedConnectionModes"][sourceType][i] + ) + connectionItemObj["enum"] = connectionModesEnum + connectionObj["properties"][sourceType] = connectionItemObj return connectionObj -def generate_schema_properties(uiConfig, dbConfig, schemaObject, properties, name, selector): +def generate_schema_properties( + uiConfig, dbConfig, schemaObject, properties, name, selector +): """Generates corresponding schema properties by iterating over each of the ui-config fields. Args: @@ -838,75 +983,98 @@ def generate_schema_properties(uiConfig, dbConfig, schemaObject, properties, nam properties (object): properties of schema name (string): name of the source or destination. selector (string): either 'source' or 'destination' - """ + """ if is_old_format(uiConfig): for group in uiConfig: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: if "preRequisiteField" in field: continue - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['value']] = generateFunction( - field, dbConfig, 'value') - if field.get('required', False) == True and is_field_present_in_default_config(field, dbConfig, "value"): - schemaObject['required'].append(field['value']) + properties[field["value"]] = generateFunction( + field, dbConfig, "value" + ) + if field.get( + "required", False + ) == True and is_field_present_in_default_config( + field, dbConfig, "value" + ): + schemaObject["required"].append(field["value"]) else: - if selector == 'destination': - baseTemplate = uiConfig.get('baseTemplate', []) - sdkTemplate = uiConfig.get('sdkTemplate', {}) - consentSettingsTemplate = uiConfig.get('consentSettingsTemplate', {}) + if selector == "destination": + baseTemplate = uiConfig.get("baseTemplate", []) + sdkTemplate = uiConfig.get("sdkTemplate", {}) + consentSettingsTemplate = uiConfig.get("consentSettingsTemplate", {}) for template in baseTemplate: - for section in template.get('sections', []): - for group in section.get('groups', []): - for field in group.get('fields', []): - generateFunction = uiTypetoSchemaFn.get( - field['type'], None) + for section in template.get("sections", []): + for group in section.get("groups", []): + for field in group.get("fields", []): + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['configKey']] = generateFunction( - field, dbConfig, 'configKey') - if template.get('title', "") == "Initial setup" and is_field_present_in_default_config(field, dbConfig, "configKey") and 'preRequisites' not in field: - schemaObject['required'].append( - field['configKey']) - - for field in sdkTemplate.get('fields', []): - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + if ( + template.get("title", "") == "Initial setup" + and is_field_present_in_default_config( + field, dbConfig, "configKey" + ) + and "preRequisites" not in field + ): + schemaObject["required"].append(field["configKey"]) + + for field in sdkTemplate.get("fields", []): + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['configKey']] = generateFunction( - field, dbConfig, 'configKey') - if field.get('required', False) == True and is_field_present_in_default_config(field, dbConfig, "configKey"): - schemaObject['required'].append(field['configKey']) - - for field in consentSettingsTemplate.get('fields', []): - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + if field.get( + "required", False + ) == True and is_field_present_in_default_config( + field, dbConfig, "configKey" + ): + schemaObject["required"].append(field["configKey"]) + + for field in consentSettingsTemplate.get("fields", []): + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['configKey']] = generateFunction( - field, dbConfig, 'configKey') - if field.get('required', False) == True and is_field_present_in_default_config(field, dbConfig, "configKey"): - schemaObject['required'].append(field['configKey']) + properties[field["configKey"]] = generateFunction( + field, dbConfig, "configKey" + ) + if field.get( + "required", False + ) == True and is_field_present_in_default_config( + field, dbConfig, "configKey" + ): + schemaObject["required"].append(field["configKey"]) # default properties in new ui-config based schemas. - schemaObject['properties']['useNativeSDK'] = generate_schema_for_checkbox({"type":"checkbox", - "value":"useNativeSDK"}, dbConfig, "value") - schemaObject['properties']['connectionMode'] = generate_connection_mode(dbConfig) + schemaObject["properties"]["useNativeSDK"] = generate_schema_for_checkbox( + {"type": "checkbox", "value": "useNativeSDK"}, dbConfig, "value" + ) + schemaObject["properties"]["connectionMode"] = generate_connection_mode( + dbConfig + ) else: # for sources def generate_config_props(config): for group in config: - fields = group.get('fields', []) + fields = group.get("fields", []) for field in fields: - generateFunction = uiTypetoSchemaFn.get( - field["type"], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: - properties[field['value']] = generateFunction( - field, dbConfig, 'value') + properties[field["value"]] = generateFunction( + field, dbConfig, "value" + ) - auth = uiConfig.get('auth', None) - config = uiConfig.get('config', []) + auth = uiConfig.get("auth", None) + config = uiConfig.get("config", []) if auth: - type = auth.get('type') + type = auth.get("type") if type == "form": - auth_config = auth.get('config', []) + auth_config = auth.get("config", []) generate_config_props(auth_config) generate_config_props(config) @@ -923,31 +1091,33 @@ def generate_schema(uiConfig, dbConfig, name, selector): Returns: object: schema - """ + """ newSchema = {} schemaObject = {} - schemaObject['$schema'] = 'http://json-schema.org/draft-07/schema#' - schemaObject['required'] = [] - schemaObject['type'] = "object" - schemaObject['properties'] = {} + schemaObject["$schema"] = "http://json-schema.org/draft-07/schema#" + schemaObject["required"] = [] + schemaObject["type"] = "object" + schemaObject["properties"] = {} allOfSchemaObj = {} if is_old_format(uiConfig): allOfSchemaObj = generate_schema_for_allOf(uiConfig, dbConfig, "value") if allOfSchemaObj: # AnyOf occurring separately, not inside allOf. if len(allOfSchemaObj) == 1: - if isinstance(allOfSchemaObj[0], list): - schemaObject['anyOf'] = allOfSchemaObj[0] - else: - schemaObject['anyOf'] = allOfSchemaObj + if isinstance(allOfSchemaObj[0], list): + schemaObject["anyOf"] = allOfSchemaObj[0] + else: + schemaObject["anyOf"] = allOfSchemaObj else: - schemaObject['allOf'] = allOfSchemaObj - generate_schema_properties(uiConfig, dbConfig, schemaObject, - schemaObject['properties'], name, selector) - newSchema['configSchema'] = schemaObject + schemaObject["allOf"] = allOfSchemaObj + generate_schema_properties( + uiConfig, dbConfig, schemaObject, schemaObject["properties"], name, selector + ) + newSchema["configSchema"] = schemaObject return newSchema + def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): """Generates warning for each schema difference created by the current ui-type. @@ -956,7 +1126,7 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): dbConfig (object): Configurations of db-config.json. schema (object): Existing schema in schema.json curUiType (string): Ui-Type for which warnings are generated. - """ + """ if is_old_format(uiConfig): for uiConfigItem in uiConfig: for field in uiConfigItem["fields"]: @@ -965,77 +1135,138 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): if field["type"] == curUiType: if field["value"] not in schema["properties"]: warnings.warn( - f'{field["value"]} field is not in schema \n', UserWarning) + f'{field["value"]} field is not in schema \n', UserWarning + ) else: curSchemaField = schema["properties"][field["value"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "value") + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "value" + ) schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["value"], get_formatted_json(schemaDiff)), UserWarning) + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, + field["value"], + get_formatted_json(schemaDiff), + ), + UserWarning, + ) else: - baseTemplate = uiConfig.get('baseTemplate', []) - sdkTemplate = uiConfig.get('sdkTemplate', {}) - consentSettingsTemplate = uiConfig.get('consentSettingsTemplate', {}) + baseTemplate = uiConfig.get("baseTemplate", []) + sdkTemplate = uiConfig.get("sdkTemplate", {}) + consentSettingsTemplate = uiConfig.get("consentSettingsTemplate", {}) for template in baseTemplate: - for section in template.get('sections', []): - for group in section.get('groups', []): + for section in template.get("sections", []): + for group in section.get("groups", []): if "preRequisites" in group: continue - for field in group.get('fields', []): + for field in group.get("fields", []): if "preRequisites" in field: continue - generateFunction = uiTypetoSchemaFn.get( - field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction and field["type"] == curUiType: if field["configKey"] not in schema["properties"]: warnings.warn( - f'{field["configKey"]} field is not in schema \n', UserWarning) + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) else: - curSchemaField = schema["properties"][field["configKey"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "configKey") - schemaDiff = get_json_diff(curSchemaField, newSchemaField) + curSchemaField = schema["properties"][ + field["configKey"] + ] + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) + schemaDiff = get_json_diff( + curSchemaField, newSchemaField + ) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) - - for field in sdkTemplate.get('fields', []): + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, + field["configKey"], + get_formatted_json(schemaDiff), + ), + UserWarning, + ) + + for field in sdkTemplate.get("fields", []): if "preRequisites" in field: continue - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: if generateFunction and field["type"] == curUiType: if field["configKey"] not in schema["properties"]: warnings.warn( - f'{field["configKey"]} field is not in schema \n', UserWarning) + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) else: curSchemaField = schema["properties"][field["configKey"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "configKey") + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) schemaDiff = get_json_diff(curSchemaField, newSchemaField) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], get_formatted_json(schemaDiff)), UserWarning) - - for field in consentSettingsTemplate.get('fields', []): + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, + field["configKey"], + get_formatted_json(schemaDiff), + ), + UserWarning, + ) + + for field in sdkTemplate.get("fields", []): + if "preRequisites" in field: + continue + generateFunction = uiTypetoSchemaFn.get(field["type"], None) + if generateFunction: + if generateFunction and field["type"] == curUiType: + if field["configKey"] not in schema["properties"]: + warnings.warn( + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) + else: + curSchemaField = schema["properties"][field["configKey"]] + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) + schemaDiff = get_json_diff(newSchemaField, curSchemaField) + if schemaDiff: + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, field["configKey"], schemaDiff + ), + UserWarning, + ) + + for field in consentSettingsTemplate.get("fields", []): if "preRequisites" in field: continue - generateFunction = uiTypetoSchemaFn.get(field['type'], None) + generateFunction = uiTypetoSchemaFn.get(field["type"], None) if generateFunction: if generateFunction and field["type"] == curUiType: if field["configKey"] not in schema["properties"]: warnings.warn( - f'{field["configKey"]} field is not in schema \n', UserWarning) + f'{field["configKey"]} field is not in schema \n', + UserWarning, + ) else: curSchemaField = schema["properties"][field["configKey"]] - newSchemaField = uiTypetoSchemaFn.get( - curUiType)(field, dbConfig, "configKey") - schemaDiff = diff(newSchemaField, curSchemaField) + newSchemaField = uiTypetoSchemaFn.get(curUiType)( + field, dbConfig, "configKey" + ) + schemaDiff = get_json_diff(newSchemaField, curSchemaField) if schemaDiff: - warnings.warn("For type:{} field:{} Difference is : \n\n {} \n".format( - curUiType, field["configKey"], schemaDiff), UserWarning) + warnings.warn( + "For type:{} field:{} Difference is : \n\n {} \n".format( + curUiType, field["configKey"], schemaDiff + ), + UserWarning, + ) uiTypetoSchemaFn = { @@ -1045,29 +1276,33 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): "textareaInput": generate_schema_for_textarea_input, "singleSelect": generate_schema_for_single_select, "dynamicCustomForm": generate_schema_for_dynamic_custom_form, - 'dynamicForm': generate_schema_for_dynamic_form, - 'mapping': generate_schema_for_mapping, - 'dynamicSelectForm': generate_schema_for_dynamic_select_form, - 'tagInput': generate_schema_for_tag_input, - 'timeRangePicker': generate_schema_for_time_range_picker, - 'timePicker': generate_schema_for_time_picker + "dynamicForm": generate_schema_for_dynamic_form, + "mapping": generate_schema_for_mapping, + "dynamicSelectForm": generate_schema_for_dynamic_select_form, + "tagInput": generate_schema_for_tag_input, + "timeRangePicker": generate_schema_for_time_range_picker, + "timePicker": generate_schema_for_time_picker, } + def save_schema_to_file(selector, name, schema): # Get the parent directory (one level up) script_directory = os.path.dirname(os.path.abspath(__file__)) directory = os.path.dirname(script_directory) # Define the relative path - relative_path = f'{CONFIG_DIR}/{selector}s/{name}/schema.json' + relative_path = f"{CONFIG_DIR}/{selector}s/{name}/schema.json" file_path = os.path.join(directory, relative_path) # Write the new content - with open(file_path, 'w') as file: + with open(file_path, "w") as file: file.write(get_formatted_json(schema)) -def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema): - """Generates a schema and compares it with an existing one. + +def validate_config_consistency( + name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema +): + """Generates a schema and compares it with an existing one. If schemaDiff is present, it calls for individual warnings by iterating over each ui-type. Args: @@ -1077,13 +1312,13 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou dbConfig (object): Configurations of db-config.json. schema (object): Existing schema in schema.json. shouldUpdateSchema (boolean): if it should update the existing schema with generated one - """ + """ if schema == None and uiConfig == None: return if uiConfig == None: - print('-'*50) + print("-" * 50) warnings.warn(f"Ui-Config is null for {name} in {selector} \n", UserWarning) - print('-'*50) + print("-" * 50) return generatedSchema = generate_schema(uiConfig, dbConfig, name, selector) @@ -1095,23 +1330,30 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou save_schema_to_file(selector, name, finalSchema) if schemaDiff: - print('-'*50) - print(f'Schema diff for {name} in {selector}s') + print("-" * 50) + print(f"Schema diff for {name} in {selector}s") # call for individual warnings for uiType in uiTypetoSchemaFn.keys(): generate_warnings_for_each_type(uiConfig, dbConfig, schema, uiType) # schema diff for "additionalProperties" if "additionalProperties" not in schema: - print("\n Recommendation: Please set additionalProperties to False in schema.json. \n") + print( + "\n Recommendation: Please set additionalProperties to False in schema.json. \n" + ) # schema diff for "required" if "required" not in schema: - warnings.warn('required field is not in schema \n', UserWarning) + warnings.warn("required field is not in schema \n", UserWarning) else: curRequiredField = schema["required"] newRequiredField = generatedSchema["configSchema"]["required"] requiredFieldDiff = get_json_diff(curRequiredField, newRequiredField) if requiredFieldDiff: - warnings.warn("For required field Difference is : \n\n {} \n".format(get_formatted_json(requiredFieldDiff)), UserWarning) + warnings.warn( + "For required field Difference is : \n\n {} \n".format( + get_formatted_json(requiredFieldDiff) + ), + UserWarning, + ) if "allOf" in generatedSchema["configSchema"]: curAllOfSchema = {} if "allOf" in schema: @@ -1119,7 +1361,12 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou newAllOfSchema = generatedSchema["configSchema"]["allOf"] allOfSchemaDiff = get_json_diff(curAllOfSchema, newAllOfSchema) if allOfSchemaDiff: - warnings.warn("For allOf field Difference is : \n\n {} \n".format(get_formatted_json(allOfSchemaDiff)), UserWarning) + warnings.warn( + "For allOf field Difference is : \n\n {} \n".format( + get_formatted_json(allOfSchemaDiff) + ), + UserWarning, + ) if "anyOf" in generatedSchema["configSchema"]: curAnyOfSchema = {} if "anyOf" in schema: @@ -1127,30 +1374,36 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shou newAnyOfSchema = generatedSchema["configSchema"]["anyOf"] anyOfSchemaDiff = get_json_diff(curAnyOfSchema, newAnyOfSchema) if anyOfSchemaDiff: - warnings.warn("For anyOf field Difference is : \n\n {} \n".format(get_formatted_json(anyOfSchemaDiff)), UserWarning) - print('-'*50) + warnings.warn( + "For anyOf field Difference is : \n\n {} \n".format( + get_formatted_json(anyOfSchemaDiff) + ), + UserWarning, + ) + print("-" * 50) else: if shouldUpdateSchema: save_schema_to_file(selector, name, generatedSchema) - print('-'*50) - print(f'Generated schema for {name} in {selector}s') + print("-" * 50) + print(f"Generated schema for {name} in {selector}s") print(get_formatted_json(generatedSchema)) - print('-'*50) + print("-" * 50) + def get_schema_diff(name, selector, shouldUpdateSchema=False): - """ Validates the schema for the given name and selector. + """Validates the schema for the given name and selector. Args: name (string): name of the source or destination. selector (string): either 'source' or 'destination'. shouldUpdateSchema (boolean): if it should update the existing schema with generated one - """ + """ - file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] - directory = f'./{CONFIG_DIR}/{selector}s/{name}' + file_selectors = ["db-config.json", "ui-config.json", "schema.json"] + directory = f"./{CONFIG_DIR}/{selector}s/{name}" if not os.path.isdir(directory): - print(f'No {selector}s directory found for {name}') + print(f"No {selector}s directory found for {name}") return if name not in EXCLUDED_DEST: @@ -1158,35 +1411,55 @@ def get_schema_diff(name, selector, shouldUpdateSchema=False): file_content = {} for file_selector in file_selectors: if file_selector in available_files: - file_content.update(get_json_from_file(f'{directory}/{file_selector}')) + file_content.update(get_json_from_file(f"{directory}/{file_selector}")) uiConfig = file_content.get("uiConfig") schema = file_content.get("configSchema") dbConfig = file_content.get("config") - validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) + validate_config_consistency( + name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema + ) + -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json') +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json" + ) group = parser.add_mutually_exclusive_group() - parser.add_argument('selector', metavar='selector', type=str, help='Enter whether -name is a source or destination') - parser.add_argument('-update', action='store_true', help='Will update existing schema with any changes') - group.add_argument('-name', metavar='name', type=str, help='Enter the folder name under selector') - group.add_argument('-all', action='store_true', help='Will run validation for all entities under selector') - + parser.add_argument( + "selector", + metavar="selector", + type=str, + help="Enter whether -name is a source or destination", + ) + parser.add_argument( + "-update", + action="store_true", + help="Will update existing schema with any changes", + ) + group.add_argument( + "-name", metavar="name", type=str, help="Enter the folder name under selector" + ) + group.add_argument( + "-all", + action="store_true", + help="Will run validation for all entities under selector", + ) + args = parser.parse_args() selector = args.selector shouldUpdateSchema = args.update if args.all: - dir_path = f'./{CONFIG_DIR}/{selector}s' + dir_path = f"./{CONFIG_DIR}/{selector}s" if not os.path.isdir(dir_path): - print(f'No {selector}s folder found') + print(f"No {selector}s folder found") exit(1) - + current_items = os.listdir(dir_path) for name in current_items: get_schema_diff(name, selector, shouldUpdateSchema) - + else: name = args.name get_schema_diff(name, selector, shouldUpdateSchema) diff --git a/scripts/setup-python.sh b/scripts/setup-python.sh index b71b35f43..e5518394d 100755 --- a/scripts/setup-python.sh +++ b/scripts/setup-python.sh @@ -5,3 +5,4 @@ pip3 --version pip3 install requests pip3 install jsonschema pip3 install jsondiff +pip3 install black diff --git a/scripts/utils.py b/scripts/utils.py index 49cc83027..b61e8cdc3 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,6 +1,7 @@ from jsondiff import JsonDiffer import json + def get_json_diff(oldJson, newJson): """Returns the difference between two JSONs. @@ -10,10 +11,11 @@ def get_json_diff(oldJson, newJson): Returns: object: difference between oldJson and newJson. - """ + """ differ = JsonDiffer(marshal=True) return differ.diff(oldJson, newJson) + def apply_json_diff(oldJson, diff): """Applies the difference on oldJson and returns the newJson. @@ -23,10 +25,11 @@ def apply_json_diff(oldJson, diff): Returns: object: new json. - """ + """ differ = JsonDiffer(marshal=True) return differ.patch(oldJson, diff) + def get_formatted_json(jsonObj): """Formats the json object. @@ -35,9 +38,10 @@ def get_formatted_json(jsonObj): Returns: string: formatted json. - """ + """ return json.dumps(jsonObj, indent=2, ensure_ascii=False) + def get_json_from_file(filePath): """Reads the content of the file and returns the json object. @@ -46,6 +50,6 @@ def get_json_from_file(filePath): Returns: object: json object. - """ - with open(filePath, 'r') as file: - return json.loads(file.read().encode('utf-8', 'ignore')) + """ + with open(filePath, "r") as file: + return json.loads(file.read().encode("utf-8", "ignore")) diff --git a/test/test_configGenerator.py b/test/test_configGenerator.py index 1964e7f77..19cf42839 100644 --- a/test/test_configGenerator.py +++ b/test/test_configGenerator.py @@ -10,20 +10,22 @@ from scripts.configGenerator import generateConfigs -with open('test/configData/inputData.json', 'r') as file: +with open("test/configData/inputData.json", "r") as file: input_data = json.load(file) -with open('test/configData/db-config.json', 'r') as file: +with open("test/configData/db-config.json", "r") as file: db_config = json.load(file) -with open('test/configData/ui-config.json', 'r') as file: +with open("test/configData/ui-config.json", "r") as file: ui_config = json.load(file) + class TestConfigGenerator(unittest.TestCase): def test_config_generator(self): result = generateConfigs(input_data) - self.assertEqual(result['db_config'], json.dumps(db_config)) - self.assertEqual(result['ui_config'], json.dumps(ui_config)) + self.assertEqual(result["db_config"], json.dumps(db_config)) + self.assertEqual(result["ui_config"], json.dumps(ui_config)) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From daf9fa8256eed280bad04c0c109de3787e6262e0 Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:51:07 +0530 Subject: [PATCH 05/20] feat: onboard new destination commandbar (#1196) --- .../destinations/commandbar/db-config.json | 39 ++++ .../destinations/commandbar/schema.json | 71 +++++++ .../destinations/commandbar/ui-config.json | 180 ++++++++++++++++++ .../validation/destinations/commandbar.json | 96 ++++++++++ 4 files changed, 386 insertions(+) create mode 100644 src/configurations/destinations/commandbar/db-config.json create mode 100644 src/configurations/destinations/commandbar/schema.json create mode 100644 src/configurations/destinations/commandbar/ui-config.json create mode 100644 test/data/validation/destinations/commandbar.json diff --git a/src/configurations/destinations/commandbar/db-config.json b/src/configurations/destinations/commandbar/db-config.json new file mode 100644 index 000000000..6decfd024 --- /dev/null +++ b/src/configurations/destinations/commandbar/db-config.json @@ -0,0 +1,39 @@ +{ + "name": "COMMANDBAR", + "displayName": "CommandBar", + "config": { + "transformAtV1": "processor", + "saveDestinationResponse": true, + "includeKeys": [ + "orgId", + "blacklistedEvents", + "whitelistedEvents", + "oneTrustCookieCategories", + "eventFilteringOption" + ], + "excludeKeys": [], + "supportedSourceTypes": ["web"], + "supportedMessageTypes": { + "device": { + "web": ["track", "identify"] + } + }, + "supportedConnectionModes": { + "web": ["device"] + }, + "destConfig": { + "defaultConfig": [ + "orgId", + "blacklistedEvents", + "whitelistedEvents", + "eventFilteringOption", + "oneTrustCookieCategories" + ], + "web": ["useNativeSDK", "connectionMode"] + }, + "secretKeys": ["orgId"] + }, + "options": { + "isBeta": true + } +} diff --git a/src/configurations/destinations/commandbar/schema.json b/src/configurations/destinations/commandbar/schema.json new file mode 100644 index 000000000..d1f050356 --- /dev/null +++ b/src/configurations/destinations/commandbar/schema.json @@ -0,0 +1,71 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["orgId"], + "type": "object", + "properties": { + "orgId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "eventFilteringOption": { + "type": "string", + "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], + "default": "disable" + }, + "whitelistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "blacklistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" + } + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { + "type": "string", + "enum": ["device"] + } + } + } + } + } +} diff --git a/src/configurations/destinations/commandbar/ui-config.json b/src/configurations/destinations/commandbar/ui-config.json new file mode 100644 index 000000000..c28a3243c --- /dev/null +++ b/src/configurations/destinations/commandbar/ui-config.json @@ -0,0 +1,180 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Organization id", + "note": "CommandBar App dashboard > Account (Lower left Corner) > Org ID", + "configKey": "orgId", + "regex": "^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Organization ID", + "placeholder": "e.g. 2***e*2*" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [], + "defaultConnectionModes": { + "web": "device" + } + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Destination settings", + "note": "Configure advanced destination-specific settings here", + "icon": "settings", + "groups": [] + }, + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "Client-side event filtering", + "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", + "preRequisites": { + "fields": [ + { + "configKey": "connectionMode.web", + "value": "device" + } + ], + "condition": "or" + }, + "fields": [ + { + "type": "singleSelect", + "label": "Choose if you want to turn on events filtering:", + "configKey": "eventFilteringOption", + "note": "You must select either allowlist or denylist to enable events filtering", + "options": [ + { + "label": "No events filtering", + "value": "disable" + }, + { + "label": "Filter via allowlist", + "value": "whitelistedEvents" + }, + { + "label": "Filter via denylist", + "value": "blacklistedEvents" + } + ], + "default": "disable" + }, + { + "type": "tagInput", + "label": "Allowlisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", + "configKey": "whitelistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "whitelistedEvents" + } + ] + } + }, + { + "type": "tagInput", + "label": "Denylisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", + "configKey": "blacklistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "blacklistedEvents" + } + ] + } + } + ] + }, + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] + } + } +} diff --git a/test/data/validation/destinations/commandbar.json b/test/data/validation/destinations/commandbar.json new file mode 100644 index 000000000..363ece4a3 --- /dev/null +++ b/test/data/validation/destinations/commandbar.json @@ -0,0 +1,96 @@ +[ + { + "config": { + "orgId": "testorgid", + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "connectionMode": { + "web": "device" + } + }, + "result": true + }, + { + "config": { + "orgId": "testorgid", + "eventFilteringOption": "whitelistedEvents", + "whitelistedEvents": [ + { + "eventName": "testevent" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": false + }, + "connectionMode": { + "web": "cloud" + } + }, + "result": false, + "err": ["connectionMode.web must be equal to one of the allowed values"] + }, + { + "config": { + "orgId": "testorgid", + "eventFilteringOption": "enable", + "whitelistedEvents": [ + { + "eventName": "testevent" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": false + }, + "connectionMode": { + "web": "device" + } + }, + "result": false, + "err": ["eventFilteringOption must be equal to one of the allowed values"] + }, + { + "config": { + "eventFilteringOption": "whitelistedEvents", + "whitelistedEvents": [ + { + "eventName": "testevent" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": false + }, + "connectionMode": { + "web": "device" + } + }, + "result": false, + "err": [" must have required property 'orgId'"] + } +] From 7faa64acfe2fca20b75789f31b541dd33e7fde8c Mon Sep 17 00:00:00 2001 From: Abhishek Pandey <64667840+1abhishekpandey@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:21:10 +0530 Subject: [PATCH 06/20] fix: comscore schema (#1217) --- src/configurations/destinations/comscore/schema.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/comscore/schema.json b/src/configurations/destinations/comscore/schema.json index 64a668270..ec6be4b99 100644 --- a/src/configurations/destinations/comscore/schema.json +++ b/src/configurations/destinations/comscore/schema.json @@ -25,7 +25,10 @@ "useNativeSDK": { "type": "object", "properties": { - "web": { + "android": { + "type": "boolean" + }, + "ios": { "type": "boolean" } } From 9124fb44fd18d903bbf37f7cef3eaf7aeccf92e4 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:37:39 +0530 Subject: [PATCH 07/20] feat: adding custom field support for freshsales (#1195) --- .../destinations/freshsales/db-config.json | 1 + .../destinations/freshsales/schema.json | 16 ++++++++++++++++ .../destinations/freshsales/ui-config.json | 13 +++++++++++++ .../validation/destinations/freshsales.json | 17 +++++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/src/configurations/destinations/freshsales/db-config.json b/src/configurations/destinations/freshsales/db-config.json index a23a997e9..3a377b063 100644 --- a/src/configurations/destinations/freshsales/db-config.json +++ b/src/configurations/destinations/freshsales/db-config.json @@ -38,6 +38,7 @@ "apiKey", "domain", "rudderEventsToFreshsalesEvents", + "customPropertyMapping", "oneTrustCookieCategories" ] } diff --git a/src/configurations/destinations/freshsales/schema.json b/src/configurations/destinations/freshsales/schema.json index a58ed3a80..d13d3de60 100644 --- a/src/configurations/destinations/freshsales/schema.json +++ b/src/configurations/destinations/freshsales/schema.json @@ -28,6 +28,22 @@ } } }, + "customPropertyMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, "oneTrustCookieCategories": { "type": "array", "items": { diff --git a/src/configurations/destinations/freshsales/ui-config.json b/src/configurations/destinations/freshsales/ui-config.json index 3aa329eec..df24401e7 100644 --- a/src/configurations/destinations/freshsales/ui-config.json +++ b/src/configurations/destinations/freshsales/ui-config.json @@ -47,6 +47,19 @@ "value": "lifecycle_stage" } ] + }, + { + "type": "dynamicForm", + "label": "Mapping to add custom fields while creating / updating a contact", + "value": "customPropertyMapping", + "required": false, + "labelRight": "Freshsales Contact Custom Field Name", + "labelLeft": "Rudder Payload Property", + "keyLeft": "from", + "keyRight": "to", + "placeholderLeft": "e.g: newTrait", + "placeholderRight": "e.g: cf_new_trait", + "footerNote": "Map Rudder Traits to Freshsales Contact Custom Fields. Here, traits will be fetched from traits object(message.traits or message.context.traits). Please provide the exact Internal name of the custom trait" } ] }, diff --git a/test/data/validation/destinations/freshsales.json b/test/data/validation/destinations/freshsales.json index 6bece2cec..df7962153 100644 --- a/test/data/validation/destinations/freshsales.json +++ b/test/data/validation/destinations/freshsales.json @@ -35,5 +35,22 @@ }, "result": false, "err": ["domain must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] + }, + { + "config": { + "apiKey": "adhs123-adhs123-adhs3", + "domain": "rudder-123457.myfreshworks.com", + "customPropertyMapping": [ + { + "from": "ofwinqeoqwefnoewqo9", + "to": "test" + }, + { + "from": "idwhcbiwdfbciwdfw", + "to": "entry" + } + ] + }, + "result": true } ] From 61d36fa5db882f00d97dd9dc88b2f26a7d49a87a Mon Sep 17 00:00:00 2001 From: Anant Jain <62471433+anantjain45823@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:06:25 +0530 Subject: [PATCH 08/20] feat: tiktok_offline_events added support for all Standard events (#1216) --- .../tiktok_ads_offline_events/schema.json | 18 ++++++++- .../tiktok_ads_offline_events/ui-config.json | 40 +++++++++++++++++++ .../tiktok_ads_offline_events.json | 17 ++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/tiktok_ads_offline_events/schema.json b/src/configurations/destinations/tiktok_ads_offline_events/schema.json index 07ed15244..bcca72c7b 100644 --- a/src/configurations/destinations/tiktok_ads_offline_events/schema.json +++ b/src/configurations/destinations/tiktok_ads_offline_events/schema.json @@ -19,7 +19,23 @@ }, "to": { "type": "string", - "enum": ["CompletePayment", "Contact", "SubmitForm", "Subscribe", ""] + "enum": [ + "CompletePayment", + "Contact", + "SubmitForm", + "Subscribe", + "AddPaymentInfo", + "AddToCart", + "AddToWishlist", + "ClickButton", + "CompleteRegistration", + "Download", + "InitiateCheckout", + "PlaceAnOrder", + "Search", + "ViewContent", + "" + ] } } } diff --git a/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json b/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json index 57427e77f..5bfd80e47 100644 --- a/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json +++ b/src/configurations/destinations/tiktok_ads_offline_events/ui-config.json @@ -44,6 +44,46 @@ { "name": "Subscribe", "value": "Subscribe" + }, + { + "name": "Add Payment Info", + "value": "AddPaymentInfo" + }, + { + "name": "Add to Cart", + "value": "AddToCart" + }, + { + "name": "Add to Wishlist", + "value": "AddToWishlist" + }, + { + "name": "Click Button", + "value": "ClickButton" + }, + { + "name": "Complete Registration", + "value": "CompleteRegistration" + }, + { + "name": "Download", + "value": "Download" + }, + { + "name": "Initiate Checkout", + "value": "InitiateCheckout" + }, + { + "name": "Place an Order", + "value": "PlaceAnOrder" + }, + { + "name": "Search", + "value": "Search" + }, + { + "name": "View Content", + "value": "ViewContent" } ] } diff --git a/test/data/validation/destinations/tiktok_ads_offline_events.json b/test/data/validation/destinations/tiktok_ads_offline_events.json index 0b26387f3..7f25d3448 100644 --- a/test/data/validation/destinations/tiktok_ads_offline_events.json +++ b/test/data/validation/destinations/tiktok_ads_offline_events.json @@ -35,12 +35,29 @@ { "from": "c", "to": "Subscribe" + }, + { + "from": "d", + "to": "Search" } ] }, "result": false, "err": [" must have required property 'accessToken'"] }, + { + "config": { + "accessToken": "fuwheirujkvjnkrtgkf", + "hashUserProperties": true, + "eventsToStandard": [ + { + "from": "a", + "to": "InitiateCheckout" + } + ] + }, + "result": true + }, { "config": { "accessToken": "fuwheirujkvjnkrtgkf", From 0d9b8d7ee45b6ba5c914aaca6c2c11474a7575e9 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Sat, 17 Feb 2024 11:19:53 +0530 Subject: [PATCH 09/20] feat: onboard trade desk real time conversions (#1213) --- .../db-config.json | 50 ++++ .../schema.json | 117 +++++++++ .../ui-config.json | 246 ++++++++++++++++++ .../the_trade_desk_real_time_conversions.json | 98 +++++++ 4 files changed, 511 insertions(+) create mode 100644 src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json create mode 100644 src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json create mode 100644 src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json create mode 100644 test/data/validation/destinations/the_trade_desk_real_time_conversions.json diff --git a/src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json b/src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json new file mode 100644 index 000000000..fa234988b --- /dev/null +++ b/src/configurations/destinations/the_trade_desk_real_time_conversions/db-config.json @@ -0,0 +1,50 @@ +{ + "name": "THE_TRADE_DESK_REAL_TIME_CONVERSIONS", + "displayName": "The Trade Desk Real Time Conversions", + "config": { + "cdkV2Enabled": true, + "transformAtV1": "processor", + "saveDestinationResponse": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova", + "shopify" + ], + "supportedMessageTypes": { + "cloud": ["track"] + }, + "supportedConnectionModes": { + "android": ["cloud"], + "ios": ["cloud"], + "web": ["cloud"], + "unity": ["cloud"], + "amp": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"] + }, + "destConfig": { + "defaultConfig": [ + "advertiserId", + "customProperties", + "eventsMapping", + "trackerId", + "oneTrustCookieCategories" + ] + } + }, + "options": { + "isBeta": true + } +} diff --git a/src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json b/src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json new file mode 100644 index 000000000..c951319a2 --- /dev/null +++ b/src/configurations/destinations/the_trade_desk_real_time_conversions/schema.json @@ -0,0 +1,117 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["advertiserId", "trackerId"], + "type": "object", + "properties": { + "advertiserId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "trackerId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "eventsMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "enum": [ + "searchitem", + "searchcategory", + "login", + "messagebusiness", + "direction", + "sitevisit" + ] + } + } + } + }, + "customProperties": { + "type": "array", + "items": { + "type": "object", + "properties": { + "rudderProperty": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "tradeDeskProperty": { + "type": "string", + "enum": ["td1", "td2", "td3", "td4", "td5", "td6", "td7", "td8", "td9", "td10"] + } + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { + "type": "string", + "enum": ["cloud"] + }, + "ios": { + "type": "string", + "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud"] + }, + "unity": { + "type": "string", + "enum": ["cloud"] + }, + "amp": { + "type": "string", + "enum": ["cloud"] + }, + "cloud": { + "type": "string", + "enum": ["cloud"] + }, + "warehouse": { + "type": "string", + "enum": ["cloud"] + }, + "reactnative": { + "type": "string", + "enum": ["cloud"] + }, + "flutter": { + "type": "string", + "enum": ["cloud"] + }, + "cordova": { + "type": "string", + "enum": ["cloud"] + }, + "shopify": { + "type": "string", + "enum": ["cloud"] + } + } + } + } + } +} diff --git a/src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json b/src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json new file mode 100644 index 000000000..60c42a304 --- /dev/null +++ b/src/configurations/destinations/the_trade_desk_real_time_conversions/ui-config.json @@ -0,0 +1,246 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Advertiser ID", + "note": "Enter your Advertiser Id from advertiser preferences page in The Trade Desk platform UI.", + "configKey": "advertiserId", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Advertiser ID", + "placeholder": "e.g: jxXXXph" + }, + { + "type": "textInput", + "label": "Tracking Tag ID", + "note": "Enter the tracking tag ID. Contact your The Trade Desk representative to obtain it.", + "configKey": "trackerId", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid Tracking Tag ID", + "placeholder": "e.g: hcXXXke" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack's OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing 'Enter' after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Event mapping", + "note": "Map RudderStack to Trade Desk events", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "RudderStack to Trade Desk event mappings", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "eventAndPropertiesMapping", + "label": "Event and property mappings", + "note": "Map RudderStack events/properties to Trade Desk events/properties" + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "SDK settings", + "note": "not visible in the ui", + "fields": [] + }, + "redirectGroups": { + "eventAndPropertiesMapping": { + "tabs": [ + { + "name": "Event", + "fields": [ + { + "type": "mapping", + "label": "Map your RudderStack Events to Trade Desk Events", + "note": "Enter the input event name which you want to map with a Trade Desk Event Name", + "configKey": "eventsMapping", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "from", + "label": "RudderStack Event", + "placeholder": "e.g: Product Searched" + }, + { + "type": "singleSelect", + "key": "to", + "label": "Trade Desk Event", + "options": [ + { + "name": "searchitem", + "value": "searchitem" + }, + { + "name": "searchcategory", + "value": "searchcategory" + }, + { + "name": "login", + "value": "login" + }, + { + "name": "messagebusiness", + "value": "messagebusiness" + }, + { + "name": "direction", + "value": "direction" + }, + { + "name": "sitevisit", + "value": "sitevisit" + } + ] + } + ] + } + ] + }, + { + "name": "Custom properties", + "fields": [ + { + "type": "mapping", + "label": "Map custom properties", + "configKey": "customProperties", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "rudderProperty", + "label": "RudderStack property path", + "placeholder": "e.g properties.key1" + }, + { + "type": "singleSelect", + "key": "tradeDeskProperty", + "label": "Trade Desk custom property", + "options": [ + { + "label": "td1", + "value": "td1" + }, + { + "label": "td2", + "value": "td2" + }, + { + "label": "td3", + "value": "td3" + }, + { + "label": "td4", + "value": "td4" + }, + { + "label": "td5", + "value": "td5" + }, + { + "label": "td6", + "value": "td6" + }, + { + "label": "td7", + "value": "td7" + }, + { + "label": "td8", + "value": "td8" + }, + { + "label": "td9", + "value": "td9" + }, + { + "label": "td10", + "value": "td10" + } + ] + } + ] + } + ] + } + ] + } + } + } +} diff --git a/test/data/validation/destinations/the_trade_desk_real_time_conversions.json b/test/data/validation/destinations/the_trade_desk_real_time_conversions.json new file mode 100644 index 000000000..dd43b1121 --- /dev/null +++ b/test/data/validation/destinations/the_trade_desk_real_time_conversions.json @@ -0,0 +1,98 @@ +[ + { + "config": { + "advertiserId": "test-advertiserId", + "trackerId": "test-trackerId", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ], + "eventsMapping": [ + { + "from": "Products Searched", + "to": "searchitem" + } + ], + "customProperties": [ + { + "rudderProperty": "properties.key1", + "tradeDeskProperty": "td1" + } + ] + }, + "result": true + }, + { + "config": { + "advertiserId": "", + "trackerId": "test-trackerId", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "advertiserId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "advertiserId": "test-advertiserId", + "trackerId": "", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + "trackerId must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [ + " must have required property 'advertiserId'", + " must have required property 'trackerId'" + ] + }, + { + "config": { + "advertiserId": "test-advertiserId", + "trackerId": "test-trackerId", + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ], + "eventsMapping": [ + { + "from": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "to": "login" + } + ], + "customProperties": [ + { + "rudderProperty": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "tradeDeskProperty": "td1" + } + ] + }, + "result": false, + "err": [ + "eventsMapping.0.from must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", + "customProperties.0.rudderProperty must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"" + ] + } +] From 5120ec5e26240f92e39b3663c090a75362157d16 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Sat, 17 Feb 2024 11:20:27 +0530 Subject: [PATCH 10/20] Revert "feat: trade desk real time conversions" (#1212) --- .../the_trade_desk/db-config.json | 33 +--- .../destinations/the_trade_desk/schema.json | 46 +----- .../the_trade_desk/ui-config.json | 154 ------------------ .../destinations/the_trade_desk.json | 21 +-- 4 files changed, 6 insertions(+), 248 deletions(-) diff --git a/src/configurations/destinations/the_trade_desk/db-config.json b/src/configurations/destinations/the_trade_desk/db-config.json index 561963d52..c0a5553c8 100644 --- a/src/configurations/destinations/the_trade_desk/db-config.json +++ b/src/configurations/destinations/the_trade_desk/db-config.json @@ -1,6 +1,6 @@ { "name": "THE_TRADE_DESK", - "displayName": "The Trade Desk", + "displayName": "The Trade Desk Audience", "config": { "cdkV2Enabled": true, "supportsBlankAudienceCreation": true, @@ -9,34 +9,12 @@ "disableJsonMapper": true, "isAudienceSupported": true, "supportsVisualMapper": true, - "supportedSourceTypes": [ - "android", - "ios", - "web", - "unity", - "amp", - "cloud", - "warehouse", - "reactnative", - "flutter", - "cordova", - "shopify" - ], + "supportedSourceTypes": ["warehouse"], "supportedMessageTypes": { - "cloud": ["record", "track"] + "cloud": ["record"] }, "supportedConnectionModes": { - "android": ["cloud"], - "ios": ["cloud"], - "web": ["cloud"], - "unity": ["cloud"], - "amp": ["cloud"], - "cloud": ["cloud"], - "warehouse": ["cloud"], - "reactnative": ["cloud"], - "flutter": ["cloud"], - "cordova": ["cloud"], - "shopify": ["cloud"] + "warehouse": ["cloud"] }, "syncBehaviours": ["mirror"], "destConfig": { @@ -44,10 +22,7 @@ "audienceId", "advertiserId", "advertiserSecretKey", - "customProperties", "dataServer", - "eventsMapping", - "trackerId", "ttlInDays", "oneTrustCookieCategories" ] diff --git a/src/configurations/destinations/the_trade_desk/schema.json b/src/configurations/destinations/the_trade_desk/schema.json index 6258c81f6..5d7bf4421 100644 --- a/src/configurations/destinations/the_trade_desk/schema.json +++ b/src/configurations/destinations/the_trade_desk/schema.json @@ -1,7 +1,7 @@ { "configSchema": { "$schema": "http://json-schema.org/draft-07/schema#", - "required": ["advertiserId", "advertiserSecretKey", "trackerId", "dataServer"], + "required": ["advertiserId", "advertiserSecretKey", "dataServer"], "type": "object", "properties": { "audienceId": { @@ -20,10 +20,6 @@ "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^([0-9]|[1-9][0-9]|1[0-7][0-9]|180)$" }, - "trackerId": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" - }, "dataServer": { "type": "string", "enum": ["apac", "tokyo", "usEastCoast", "usWestCoast", "ukEu", "china"], @@ -44,49 +40,9 @@ "connectionMode": { "type": "object", "properties": { - "android": { - "type": "string", - "enum": ["cloud"] - }, - "ios": { - "type": "string", - "enum": ["cloud"] - }, - "web": { - "type": "string", - "enum": ["cloud"] - }, - "unity": { - "type": "string", - "enum": ["cloud"] - }, - "amp": { - "type": "string", - "enum": ["cloud"] - }, - "cloud": { - "type": "string", - "enum": ["cloud"] - }, "warehouse": { "type": "string", "enum": ["cloud"] - }, - "reactnative": { - "type": "string", - "enum": ["cloud"] - }, - "flutter": { - "type": "string", - "enum": ["cloud"] - }, - "cordova": { - "type": "string", - "enum": ["cloud"] - }, - "shopify": { - "type": "string", - "enum": ["cloud"] } } } diff --git a/src/configurations/destinations/the_trade_desk/ui-config.json b/src/configurations/destinations/the_trade_desk/ui-config.json index a45abd74b..64cb50bb1 100644 --- a/src/configurations/destinations/the_trade_desk/ui-config.json +++ b/src/configurations/destinations/the_trade_desk/ui-config.json @@ -31,15 +31,6 @@ "placeholder": "e.g: u8a7f3k9p2nXXXXX4q5r2m8d7z1o9", "secret": true }, - { - "type": "textInput", - "label": "Tracking Tag ID", - "note": "Tracking tag ID from advertiser preferences page in the TTD platform UI. If you can't find your tracking tag ID, contact your Technical Account Manager.", - "configKey": "trackerId", - "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", - "regexErrorMessage": "Invalid Tracking Tag ID", - "placeholder": "e.g: hcXXXke" - }, { "type": "singleSelect", "label": "Data Server", @@ -156,157 +147,12 @@ ] } ] - }, - { - "title": "Event mapping", - "note": "Map RudderStack to Trade Desk events", - "hideEditIcon": true, - "sections": [ - { - "groups": [ - { - "title": "RudderStack to Trade Desk event mappings", - "fields": [ - { - "type": "redirect", - "redirectGroupKey": "eventAndPropertiesMapping", - "label": "Event and property mappings", - "note": "Map RudderStack events/properties to Trade Desk events/properties" - } - ] - } - ] - } - ] } ], "sdkTemplate": { "title": "SDK settings", "note": "not visible in the ui", "fields": [] - }, - "redirectGroups": { - "eventAndPropertiesMapping": { - "tabs": [ - { - "name": "Event", - "fields": [ - { - "type": "mapping", - "label": "Map your RudderStack Events to Trade Desk Events", - "note": "Enter the input event name which you want to map with a Trade Desk Event Name", - "configKey": "eventsMapping", - "default": [], - "columns": [ - { - "type": "textInput", - "key": "from", - "label": "RudderStack Event", - "placeholder": "e.g: Product Searched" - }, - { - "type": "singleSelect", - "key": "to", - "label": "Trade Desk Event", - "options": [ - { - "name": "searchitem", - "value": "searchitem" - }, - { - "name": "searchcategory", - "value": "searchcategory" - }, - { - "name": "login", - "value": "login" - }, - { - "name": "messagebusiness", - "value": "messagebusiness" - }, - { - "name": "direction", - "value": "direction" - }, - { - "name": "sitevisit", - "value": "sitevisit" - } - ] - } - ] - } - ] - }, - { - "name": "Custom properties", - "fields": [ - { - "type": "mapping", - "label": "Map custom properties", - "configKey": "customProperties", - "default": [], - "columns": [ - { - "type": "textInput", - "key": "rudderProperty", - "label": "RudderStack property path", - "placeholder": "e.g properties.key1" - }, - { - "type": "singleSelect", - "key": "tradeDeskProperty", - "label": "Trade Desk custom property", - "options": [ - { - "label": "td1", - "value": "td1" - }, - { - "label": "td2", - "value": "td2" - }, - { - "label": "td3", - "value": "td3" - }, - { - "label": "td4", - "value": "td4" - }, - { - "label": "td5", - "value": "td5" - }, - { - "label": "td6", - "value": "td6" - }, - { - "label": "td7", - "value": "td7" - }, - { - "label": "td8", - "value": "td8" - }, - { - "label": "td9", - "value": "td9" - }, - { - "label": "td10", - "value": "td10" - } - ] - } - ] - } - ] - } - ] - } } } } diff --git a/test/data/validation/destinations/the_trade_desk.json b/test/data/validation/destinations/the_trade_desk.json index 78d75059e..349b2c492 100644 --- a/test/data/validation/destinations/the_trade_desk.json +++ b/test/data/validation/destinations/the_trade_desk.json @@ -4,25 +4,12 @@ "audienceId": "test-segment", "advertiserId": "test-advertiserId", "advertiserSecretKey": "test-advertiserSecretKey", - "trackerId": "test-trackerId", "segmentName": "test-segment", "dataServer": "usEastCoast", "oneTrustCookieCategories": [ { "oneTrustCookieCategory": "Marketing" } - ], - "eventsMapping": [ - { - "from": "Product Added", - "to": "addToCart" - } - ], - "customProperties": [ - { - "rudderProperty": "properties.key1", - "tradeDeskProperty": "td1" - } ] }, "result": true @@ -31,7 +18,6 @@ "config": { "audienceId": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", "advertiserId": "test-advertiserId", - "trackerId": "test-trackerId", "advertiserSecretKey": "test-advertiserSecretKey", "segmentName": "test-segment", "dataServer": "usEastCoast", @@ -50,7 +36,6 @@ "config": { "audienceId": "test-segment", "advertiserId": "", - "trackerId": "test-trackerId", "advertiserSecretKey": "test-advertiserSecretKey", "segmentName": "test-segment", "dataServer": "usEastCoast", @@ -69,7 +54,6 @@ "config": { "audienceId": "test-segment", "advertiserId": "test-advertiserId", - "trackerId": "test-trackerId", "advertiserSecretKey": "test-advertiserSecretKey", "segmentName": "test-segment", "dataServer": "test-server", @@ -95,9 +79,6 @@ ] }, "result": false, - "err": [ - " must have required property 'advertiserSecretKey'", - " must have required property 'trackerId'" - ] + "err": [" must have required property 'advertiserSecretKey'"] } ] From dde686cc72e0ac7423487d566e8f9715cc43d24f Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Mon, 19 Feb 2024 10:52:02 +0530 Subject: [PATCH 11/20] feat: add connectionMode for all sourceTypes in dest def for selected destinations (#1197) --- .../destinations/autopilot/db-config.json | 17 ++++++++++++-- .../destinations/custify/db-config.json | 17 ++++++++++++-- .../destinations/custify/schema.json | 16 +++++++++++++ .../destinations/dynamic_yield/db-config.json | 17 ++++++++++++-- .../destinations/dynamic_yield/schema.json | 18 +++++++++++---- .../fb_custom_audience/db-config.json | 9 +++++--- .../destinations/firebase/db-config.json | 10 ++++---- .../db-config.json | 17 ++++++++++++-- .../db-config.json | 8 +++++-- .../destinations/hotjar/db-config.json | 2 +- .../destinations/hotjar/schema.json | 6 +++++ .../destinations/keen/db-config.json | 16 +++++++++++-- .../destinations/leanplum/db-config.json | 14 +++++++++-- .../destinations/leanplum/schema.json | 23 +++++++++---------- .../destinations/one_signal/db-config.json | 17 ++++++++++++-- .../destinations/one_signal/schema.json | 16 +++++++++++++ .../destinations/pinterest_tag/db-config.json | 16 +++++++++++-- .../destinations/pinterest_tag/schema.json | 16 +++++++++++++ .../destinations/pipedrive/db-config.json | 17 ++++++++++++-- .../destinations/revenue_cat/db-config.json | 17 ++++++++++++-- .../destinations/revenue_cat/schema.json | 16 +++++++++++++ .../destinations/satismeter/db-config.json | 2 +- .../destinations/satismeter/schema.json | 6 +++++ .../destinations/serenytics/db-config.json | 17 ++++++++++++-- .../destinations/serenytics/schema.json | 16 +++++++++++++ .../destinations/stormly/db-config.json | 17 ++++++++++++-- .../destinations/stormly/schema.json | 16 +++++++++++++ .../destinations/tiktok_ads/db-config.json | 8 +++++-- .../destinations/tiktok_ads/schema.json | 9 ++++++++ .../destinations/tvsquared/db-config.json | 2 +- .../destinations/vero/db-config.json | 16 +++++++++++-- .../destinations/vero/schema.json | 16 +++++++++++++ 32 files changed, 373 insertions(+), 57 deletions(-) diff --git a/src/configurations/destinations/autopilot/db-config.json b/src/configurations/destinations/autopilot/db-config.json index 3d062faa9..a90df8093 100644 --- a/src/configurations/destinations/autopilot/db-config.json +++ b/src/configurations/destinations/autopilot/db-config.json @@ -32,10 +32,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "triggerId", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "triggerId", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/custify/db-config.json b/src/configurations/destinations/custify/db-config.json index f82d60825..85eee2705 100644 --- a/src/configurations/destinations/custify/db-config.json +++ b/src/configurations/destinations/custify/db-config.json @@ -31,10 +31,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "sendAnonymousId", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "sendAnonymousId", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/custify/schema.json b/src/configurations/destinations/custify/schema.json index b1f6fc1d8..3c275c973 100644 --- a/src/configurations/destinations/custify/schema.json +++ b/src/configurations/destinations/custify/schema.json @@ -23,6 +23,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/dynamic_yield/db-config.json b/src/configurations/destinations/dynamic_yield/db-config.json index e759187e0..17ff16291 100644 --- a/src/configurations/destinations/dynamic_yield/db-config.json +++ b/src/configurations/destinations/dynamic_yield/db-config.json @@ -29,13 +29,26 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "supportedMessageTypes": { "cloud": ["identify", "track"] }, "destConfig": { - "defaultConfig": ["apiKey", "hashEmail", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "hashEmail", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/dynamic_yield/schema.json b/src/configurations/destinations/dynamic_yield/schema.json index 3016808d8..3ad1e1d96 100644 --- a/src/configurations/destinations/dynamic_yield/schema.json +++ b/src/configurations/destinations/dynamic_yield/schema.json @@ -24,12 +24,22 @@ } } }, - "useNativeSDK": { - "type": "boolean" - }, + "useNativeSDK": { "type": "boolean" }, "connectionMode": { "type": "object", - "properties": {} + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/fb_custom_audience/db-config.json b/src/configurations/destinations/fb_custom_audience/db-config.json index ca96134eb..a3022d2ff 100644 --- a/src/configurations/destinations/fb_custom_audience/db-config.json +++ b/src/configurations/destinations/fb_custom_audience/db-config.json @@ -16,7 +16,9 @@ }, "isAudienceSupported": true, "supportedConnectionModes": { - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -30,8 +32,9 @@ "subType", "oneTrustCookieCategories" ], - "cloud": ["audienceId"], - "warehouse": ["adAccountId"] + "cloud": ["audienceId", "connectionMode"], + "warehouse": ["adAccountId", "connectionMode"], + "shopify": ["connectionMode"] }, "secretKeys": ["accessToken"] }, diff --git a/src/configurations/destinations/firebase/db-config.json b/src/configurations/destinations/firebase/db-config.json index bd2f58612..2c3a0dccf 100644 --- a/src/configurations/destinations/firebase/db-config.json +++ b/src/configurations/destinations/firebase/db-config.json @@ -35,11 +35,11 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "android": ["useNativeSDK"], - "ios": ["useNativeSDK"], - "unity": ["useNativeSDK"], - "reactnative": ["useNativeSDK"], - "flutter": ["useNativeSDK"] + "android": ["useNativeSDK", "connectionMode"], + "ios": ["useNativeSDK", "connectionMode"], + "unity": ["useNativeSDK", "connectionMode"], + "reactnative": ["useNativeSDK", "connectionMode"], + "flutter": ["useNativeSDK", "connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json b/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json index 30e4ffd28..3f772d895 100644 --- a/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json +++ b/src/configurations/destinations/google_adwords_enhanced_conversions/db-config.json @@ -37,7 +37,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -48,7 +50,18 @@ "loginCustomerId", "requireHash", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json b/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json index d542e29e1..eef9de173 100644 --- a/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json +++ b/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json @@ -20,7 +20,9 @@ "cloud": ["audiencelist"] }, "supportedConnectionModes": { - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "isAudienceSupported": true, "supportsBlankAudienceCreation": true, @@ -36,7 +38,9 @@ "typeOfList", "oneTrustCookieCategories" ], - "cloud": ["audienceId"] + "cloud": ["audienceId", "connectionMode"], + "shopify": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/hotjar/db-config.json b/src/configurations/destinations/hotjar/db-config.json index ecea5199a..ca7cb05e8 100644 --- a/src/configurations/destinations/hotjar/db-config.json +++ b/src/configurations/destinations/hotjar/db-config.json @@ -29,7 +29,7 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/hotjar/schema.json b/src/configurations/destinations/hotjar/schema.json index b08b921e4..8e6042e66 100644 --- a/src/configurations/destinations/hotjar/schema.json +++ b/src/configurations/destinations/hotjar/schema.json @@ -56,6 +56,12 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["device"] } + } } } } diff --git a/src/configurations/destinations/keen/db-config.json b/src/configurations/destinations/keen/db-config.json index 22d19b035..5e6c8db77 100644 --- a/src/configurations/destinations/keen/db-config.json +++ b/src/configurations/destinations/keen/db-config.json @@ -45,7 +45,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -60,7 +62,17 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/leanplum/db-config.json b/src/configurations/destinations/leanplum/db-config.json index 67212a53a..9deb04cb3 100644 --- a/src/configurations/destinations/leanplum/db-config.json +++ b/src/configurations/destinations/leanplum/db-config.json @@ -38,7 +38,9 @@ "amp": ["cloud"], "reactnative": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "hybridModeCloudEventsFilter": { "android": { @@ -71,7 +73,15 @@ ], "android": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"], "ios": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"], - "flutter": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"] + "flutter": ["useNativeSDK", "useNativeSDKToSend", "connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["clientKey"] } diff --git a/src/configurations/destinations/leanplum/schema.json b/src/configurations/destinations/leanplum/schema.json index 8aeedf145..84a970acd 100644 --- a/src/configurations/destinations/leanplum/schema.json +++ b/src/configurations/destinations/leanplum/schema.json @@ -74,18 +74,17 @@ "connectionMode": { "type": "object", "properties": { - "android": { - "type": "string", - "enum": ["cloud", "device", "hybrid"] - }, - "ios": { - "type": "string", - "enum": ["cloud", "device", "hybrid"] - }, - "flutter": { - "type": "string", - "enum": ["cloud", "device", "hybrid"] - } + "android": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, + "ios": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, + "flutter": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } } } } diff --git a/src/configurations/destinations/one_signal/db-config.json b/src/configurations/destinations/one_signal/db-config.json index ea6d015ca..69565f1b2 100644 --- a/src/configurations/destinations/one_signal/db-config.json +++ b/src/configurations/destinations/one_signal/db-config.json @@ -31,7 +31,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -41,7 +43,18 @@ "eventAsTags", "allowedProperties", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/one_signal/schema.json b/src/configurations/destinations/one_signal/schema.json index 2d498cfdd..26fbc30b2 100644 --- a/src/configurations/destinations/one_signal/schema.json +++ b/src/configurations/destinations/one_signal/schema.json @@ -43,6 +43,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/pinterest_tag/db-config.json b/src/configurations/destinations/pinterest_tag/db-config.json index 54cdb9925..c08f2c52c 100644 --- a/src/configurations/destinations/pinterest_tag/db-config.json +++ b/src/configurations/destinations/pinterest_tag/db-config.json @@ -40,7 +40,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "supportedMessageTypes": { "cloud": ["page", "screen", "track"], @@ -69,7 +71,17 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["adAccountId", "conversionToken"] } diff --git a/src/configurations/destinations/pinterest_tag/schema.json b/src/configurations/destinations/pinterest_tag/schema.json index fb53bd829..8cf54c9be 100644 --- a/src/configurations/destinations/pinterest_tag/schema.json +++ b/src/configurations/destinations/pinterest_tag/schema.json @@ -128,6 +128,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["cloud", "device"] }, + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } }, "allOf": [ diff --git a/src/configurations/destinations/pipedrive/db-config.json b/src/configurations/destinations/pipedrive/db-config.json index a302a4d02..6328be1df 100644 --- a/src/configurations/destinations/pipedrive/db-config.json +++ b/src/configurations/destinations/pipedrive/db-config.json @@ -27,7 +27,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -39,7 +41,18 @@ "leadsMap", "organizationMap", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["apiToken"] }, diff --git a/src/configurations/destinations/revenue_cat/db-config.json b/src/configurations/destinations/revenue_cat/db-config.json index b31c28563..087037cac 100644 --- a/src/configurations/destinations/revenue_cat/db-config.json +++ b/src/configurations/destinations/revenue_cat/db-config.json @@ -31,10 +31,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "xPlatform", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "xPlatform", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/revenue_cat/schema.json b/src/configurations/destinations/revenue_cat/schema.json index 1aff6af15..098ed9a05 100644 --- a/src/configurations/destinations/revenue_cat/schema.json +++ b/src/configurations/destinations/revenue_cat/schema.json @@ -24,6 +24,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/satismeter/db-config.json b/src/configurations/destinations/satismeter/db-config.json index 5a29f4fee..b231346be 100644 --- a/src/configurations/destinations/satismeter/db-config.json +++ b/src/configurations/destinations/satismeter/db-config.json @@ -39,7 +39,7 @@ "blacklistedEvents", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"] }, "secretKeys": ["writeKey"] } diff --git a/src/configurations/destinations/satismeter/schema.json b/src/configurations/destinations/satismeter/schema.json index 0307101d2..0aa2b3802 100644 --- a/src/configurations/destinations/satismeter/schema.json +++ b/src/configurations/destinations/satismeter/schema.json @@ -64,6 +64,12 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["device"] } + } } }, "allOf": [ diff --git a/src/configurations/destinations/serenytics/db-config.json b/src/configurations/destinations/serenytics/db-config.json index fd7afe798..da2b44662 100644 --- a/src/configurations/destinations/serenytics/db-config.json +++ b/src/configurations/destinations/serenytics/db-config.json @@ -31,7 +31,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -43,7 +45,18 @@ "storageUrlPage", "storageUrlScreen", "oneTrustCookieCategories" - ] + ], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/serenytics/schema.json b/src/configurations/destinations/serenytics/schema.json index a55f77506..7574e324e 100644 --- a/src/configurations/destinations/serenytics/schema.json +++ b/src/configurations/destinations/serenytics/schema.json @@ -55,6 +55,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/stormly/db-config.json b/src/configurations/destinations/stormly/db-config.json index 01081a8fa..62c4e45c8 100644 --- a/src/configurations/destinations/stormly/db-config.json +++ b/src/configurations/destinations/stormly/db-config.json @@ -31,10 +31,23 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { - "defaultConfig": ["apiKey", "oneTrustCookieCategories"] + "defaultConfig": ["apiKey", "oneTrustCookieCategories"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "web": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] }, "secretKeys": ["apiKey"] }, diff --git a/src/configurations/destinations/stormly/schema.json b/src/configurations/destinations/stormly/schema.json index 013f7d0d4..37a30b251 100644 --- a/src/configurations/destinations/stormly/schema.json +++ b/src/configurations/destinations/stormly/schema.json @@ -19,6 +19,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "web": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/tiktok_ads/db-config.json b/src/configurations/destinations/tiktok_ads/db-config.json index ed7c780fe..cabf70b5a 100644 --- a/src/configurations/destinations/tiktok_ads/db-config.json +++ b/src/configurations/destinations/tiktok_ads/db-config.json @@ -38,7 +38,8 @@ "supportedConnectionModes": { "web": ["cloud", "device"], "android": ["cloud"], - "ios": ["cloud"] + "ios": ["cloud"], + "cloud": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -54,7 +55,10 @@ "oneTrustCookieCategories", "ketchConsentPurposes" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "cloud": ["connectionMode"] }, "secretKeys": ["accessToken", "pixelCode"] } diff --git a/src/configurations/destinations/tiktok_ads/schema.json b/src/configurations/destinations/tiktok_ads/schema.json index 85566c0aa..212a7f252 100644 --- a/src/configurations/destinations/tiktok_ads/schema.json +++ b/src/configurations/destinations/tiktok_ads/schema.json @@ -117,6 +117,15 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["cloud", "device"] }, + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] } + } } } } diff --git a/src/configurations/destinations/tvsquared/db-config.json b/src/configurations/destinations/tvsquared/db-config.json index 9855ce67b..739a32616 100644 --- a/src/configurations/destinations/tvsquared/db-config.json +++ b/src/configurations/destinations/tvsquared/db-config.json @@ -35,7 +35,7 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "connectionMode"] }, "secretKeys": ["credentials"] } diff --git a/src/configurations/destinations/vero/db-config.json b/src/configurations/destinations/vero/db-config.json index 97a7b1b8a..b9d5f1c0d 100644 --- a/src/configurations/destinations/vero/db-config.json +++ b/src/configurations/destinations/vero/db-config.json @@ -40,7 +40,9 @@ "reactnative": ["cloud"], "flutter": ["cloud"], "cordova": ["cloud"], - "shopify": ["cloud"] + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -50,7 +52,17 @@ "eventFilteringOption", "oneTrustCookieCategories" ], - "web": ["apiKey", "useNativeSDK"] + "web": ["apiKey", "useNativeSDK", "connectionMode"], + "android": ["connectionMode"], + "ios": ["connectionMode"], + "unity": ["connectionMode"], + "amp": ["connectionMode"], + "reactnative": ["connectionMode"], + "flutter": ["connectionMode"], + "cordova": ["connectionMode"], + "shopify": ["connectionMode"], + "cloud": ["connectionMode"], + "warehouse": ["connectionMode"] } } } diff --git a/src/configurations/destinations/vero/schema.json b/src/configurations/destinations/vero/schema.json index 151bf6936..7e5713120 100644 --- a/src/configurations/destinations/vero/schema.json +++ b/src/configurations/destinations/vero/schema.json @@ -65,6 +65,22 @@ } } } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { "type": "string", "enum": ["cloud", "device"] }, + "android": { "type": "string", "enum": ["cloud"] }, + "ios": { "type": "string", "enum": ["cloud"] }, + "unity": { "type": "string", "enum": ["cloud"] }, + "amp": { "type": "string", "enum": ["cloud"] }, + "reactnative": { "type": "string", "enum": ["cloud"] }, + "flutter": { "type": "string", "enum": ["cloud"] }, + "cordova": { "type": "string", "enum": ["cloud"] }, + "shopify": { "type": "string", "enum": ["cloud"] }, + "cloud": { "type": "string", "enum": ["cloud"] }, + "warehouse": { "type": "string", "enum": ["cloud"] } + } } } } From 185935467899222b8766c63c63b934d70410cc9a Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 19 Feb 2024 11:06:05 +0530 Subject: [PATCH 12/20] fix: add support of placing properties at root in af (#1203) --- src/configurations/destinations/af/db-config.json | 1 + src/configurations/destinations/af/schema.json | 4 ++++ src/configurations/destinations/af/ui-config.json | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/src/configurations/destinations/af/db-config.json b/src/configurations/destinations/af/db-config.json index 8899d8812..cb558637a 100644 --- a/src/configurations/destinations/af/db-config.json +++ b/src/configurations/destinations/af/db-config.json @@ -52,6 +52,7 @@ "devKey", "sharingFilter", "useRichEventName", + "addPropertiesAtRoot", "androidAppId", "appleAppId", "blacklistedEvents", diff --git a/src/configurations/destinations/af/schema.json b/src/configurations/destinations/af/schema.json index 9109c1dbe..12862da79 100644 --- a/src/configurations/destinations/af/schema.json +++ b/src/configurations/destinations/af/schema.json @@ -20,6 +20,10 @@ "type": "boolean", "default": false }, + "addPropertiesAtRoot": { + "type": "boolean", + "default": false + }, "sharingFilter": { "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" diff --git a/src/configurations/destinations/af/ui-config.json b/src/configurations/destinations/af/ui-config.json index 7945e3e29..cb0fe394a 100644 --- a/src/configurations/destinations/af/ui-config.json +++ b/src/configurations/destinations/af/ui-config.json @@ -39,6 +39,13 @@ "footerNote": "To include screen/page name in Screen/Page event names (ex: Viewed Contacts Page)", "default": false }, + { + "type": "checkbox", + "label": "Add properties at root in eventValue", + "value": "addPropertiesAtRoot", + "footerNote": "To send the custom properties to the root of eventValue.", + "default": false + }, { "type": "textInput", "label": "Sharing Filter", From b75e2e5aceb568345d9181b11a3ff597fb9aeeb8 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:44:38 +0530 Subject: [PATCH 13/20] Revert "feat: adding custom field support for freshsales" (#1218) --- .../destinations/freshsales/db-config.json | 1 - .../destinations/freshsales/schema.json | 16 ---------------- .../destinations/freshsales/ui-config.json | 13 ------------- .../validation/destinations/freshsales.json | 17 ----------------- 4 files changed, 47 deletions(-) diff --git a/src/configurations/destinations/freshsales/db-config.json b/src/configurations/destinations/freshsales/db-config.json index 3a377b063..a23a997e9 100644 --- a/src/configurations/destinations/freshsales/db-config.json +++ b/src/configurations/destinations/freshsales/db-config.json @@ -38,7 +38,6 @@ "apiKey", "domain", "rudderEventsToFreshsalesEvents", - "customPropertyMapping", "oneTrustCookieCategories" ] } diff --git a/src/configurations/destinations/freshsales/schema.json b/src/configurations/destinations/freshsales/schema.json index d13d3de60..a58ed3a80 100644 --- a/src/configurations/destinations/freshsales/schema.json +++ b/src/configurations/destinations/freshsales/schema.json @@ -28,22 +28,6 @@ } } }, - "customPropertyMapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - }, - "to": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" - } - } - } - }, "oneTrustCookieCategories": { "type": "array", "items": { diff --git a/src/configurations/destinations/freshsales/ui-config.json b/src/configurations/destinations/freshsales/ui-config.json index df24401e7..3aa329eec 100644 --- a/src/configurations/destinations/freshsales/ui-config.json +++ b/src/configurations/destinations/freshsales/ui-config.json @@ -47,19 +47,6 @@ "value": "lifecycle_stage" } ] - }, - { - "type": "dynamicForm", - "label": "Mapping to add custom fields while creating / updating a contact", - "value": "customPropertyMapping", - "required": false, - "labelRight": "Freshsales Contact Custom Field Name", - "labelLeft": "Rudder Payload Property", - "keyLeft": "from", - "keyRight": "to", - "placeholderLeft": "e.g: newTrait", - "placeholderRight": "e.g: cf_new_trait", - "footerNote": "Map Rudder Traits to Freshsales Contact Custom Fields. Here, traits will be fetched from traits object(message.traits or message.context.traits). Please provide the exact Internal name of the custom trait" } ] }, diff --git a/test/data/validation/destinations/freshsales.json b/test/data/validation/destinations/freshsales.json index df7962153..6bece2cec 100644 --- a/test/data/validation/destinations/freshsales.json +++ b/test/data/validation/destinations/freshsales.json @@ -35,22 +35,5 @@ }, "result": false, "err": ["domain must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] - }, - { - "config": { - "apiKey": "adhs123-adhs123-adhs3", - "domain": "rudder-123457.myfreshworks.com", - "customPropertyMapping": [ - { - "from": "ofwinqeoqwefnoewqo9", - "to": "test" - }, - { - "from": "idwhcbiwdfbciwdfw", - "to": "entry" - } - ] - }, - "result": true } ] From 6d5e40fdb812aa4c46ea13cbae5f7eb696d0cc0c Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 19 Feb 2024 13:00:22 +0530 Subject: [PATCH 14/20] feat: onboarding bluecore integration (#1182) --- .../destinations/bluecore/db-config.json | 44 +++++ .../destinations/bluecore/schema.json | 49 ++++++ .../destinations/bluecore/ui-config.json | 165 ++++++++++++++++++ .../salesforce_oauth/db-config.json | 4 +- .../validation/destinations/bluecore.json | 142 +++++++++++++++ 5 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 src/configurations/destinations/bluecore/db-config.json create mode 100644 src/configurations/destinations/bluecore/schema.json create mode 100644 src/configurations/destinations/bluecore/ui-config.json create mode 100644 test/data/validation/destinations/bluecore.json diff --git a/src/configurations/destinations/bluecore/db-config.json b/src/configurations/destinations/bluecore/db-config.json new file mode 100644 index 000000000..2d2b41218 --- /dev/null +++ b/src/configurations/destinations/bluecore/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "BLUECORE", + "displayName": "Bluecore", + "config": { + "cdkV2Enabled": true, + "transformAtV1": "processor", + "saveDestinationResponse": true, + "includeKeys": ["oneTrustCookieCategories"], + "excludeKeys": [], + "supportedSourceTypes": [ + "android", + "ios", + "unity", + "amp", + "reactnative", + "flutter", + "cordova", + "web", + "cloud", + "shopify", + "warehouse" + ], + "supportedConnectionModes": { + "android": ["cloud"], + "ios": ["cloud"], + "web": ["cloud"], + "unity": ["cloud"], + "amp": ["cloud"], + "reactnative": ["cloud"], + "flutter": ["cloud"], + "cordova": ["cloud"], + "shopify": ["cloud"], + "cloud": ["cloud"], + "warehouse": ["cloud"] + }, + "supportedMessageTypes": { + "cloud": ["identify", "track"] + }, + "destConfig": { + "defaultConfig": ["bluecoreNamespace", "eventsMapping", "oneTrustCookieCategories"] + }, + "secretKeys": ["bluecoreNamespace"] + } +} diff --git a/src/configurations/destinations/bluecore/schema.json b/src/configurations/destinations/bluecore/schema.json new file mode 100644 index 000000000..077b131ab --- /dev/null +++ b/src/configurations/destinations/bluecore/schema.json @@ -0,0 +1,49 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["bluecoreNamespace"], + "type": "object", + "properties": { + "bluecoreNamespace": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "eventsMapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + }, + "to": { + "type": "string", + "enum": [ + "viewed_product", + "search", + "add_to_cart", + "remove_from_cart", + "purchase", + "wishlist", + "" + ] + } + } + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + } + } +} diff --git a/src/configurations/destinations/bluecore/ui-config.json b/src/configurations/destinations/bluecore/ui-config.json new file mode 100644 index 000000000..6e6ef1982 --- /dev/null +++ b/src/configurations/destinations/bluecore/ui-config.json @@ -0,0 +1,165 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "Connection Settings", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "Bluecore namespace", + "note": "Get it from the url https://app.bluecore.com/admin/dashboard/", + "configKey": "bluecoreNamespace", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "secret": true, + "regexErrorMessage": "Invalid Namespace", + "placeholder": "e.g. dummy_namespace" + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Credit card visit", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + }, + { + "title": "Mappings", + "hideEditIcon": true, + "sections": [ + { + "groups": [ + { + "title": "Rudderstack to Bluecore Event Mappings", + "fields": [ + { + "type": "redirect", + "redirectGroupKey": "bluecoreEventMapping", + "label": "Event mappings" + } + ] + } + ] + } + ] + } + ], + "redirectGroups": { + "bluecoreEventMapping": { + "fields": [ + { + "type": "mapping", + "label": "Mapping to trigger the Rudderstack events with standard Bluecore events", + "configKey": "eventsMapping", + "default": [], + "columns": [ + { + "type": "textInput", + "key": "from", + "label": "Event Name", + "placeholder": "e.g: Custom Purchase Event" + }, + { + "type": "singleSelect", + "key": "to", + "label": "Bluecore Standard Ecommerce Event", + "placeholder": "e.g: purchase", + "options": [ + { + "name": "Viewed Product", + "value": "viewed_product" + }, + { + "name": "Search", + "value": "search" + }, + { + "name": "Add to Cart", + "value": "add_to_cart" + }, + { + "name": "Remove From Cart", + "value": "remove_from_cart" + }, + { + "name": "Wishlist", + "value": "wishlist" + }, + { + "name": "Purchase", + "value": "purchase" + } + ] + } + ] + } + ] + } + }, + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] + } + } +} diff --git a/src/configurations/destinations/salesforce_oauth/db-config.json b/src/configurations/destinations/salesforce_oauth/db-config.json index f01b6e89b..73934db2d 100644 --- a/src/configurations/destinations/salesforce_oauth/db-config.json +++ b/src/configurations/destinations/salesforce_oauth/db-config.json @@ -37,7 +37,5 @@ }, "secretKeys": [] }, - "options": { - "hidden": true - } + "options": {} } diff --git a/test/data/validation/destinations/bluecore.json b/test/data/validation/destinations/bluecore.json new file mode 100644 index 000000000..00e65b799 --- /dev/null +++ b/test/data/validation/destinations/bluecore.json @@ -0,0 +1,142 @@ +[ + { + "config": { + "bluecoreNamespace": "", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": false, + "err": [ + "bluecoreNamespace must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "enableAliasCall": false, + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": false, + "err": [" must have required property 'bluecoreNamespace'"] + }, + { + "config": { + "bluecoreNamespace": 1234, + "eventsMapping": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "enableAliasCall": true, + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": false, + "err": ["bluecoreNamespace must be string"] + }, + { + "config": { + "bluecoreNamespace": "qwerty1234qwerty", + "eventsToSpotifyPixelEvents": [ + { + "from": "Order Completed", + "to": "purchase" + } + ], + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + } + }, + "result": true + }, + { + "config": { + "bluecoreNamespace": "qwerty1234qwerty", + "eventsMapping": [ + { + "from": "Order Completed", + "to": "random" + } + ], + "eventFilteringOption": "disable", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "connectionMode": { + "web": "device" + } + }, + "result": false, + "err": ["eventsMapping.0.to must be equal to one of the allowed values"] + } +] From e73981f54b757aa60181942a19e7db97aac40658 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 19 Feb 2024 13:10:22 +0530 Subject: [PATCH 15/20] fix: reverting unhide of salesforce oauth (#1220) --- .../destinations/salesforce_oauth/db-config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/salesforce_oauth/db-config.json b/src/configurations/destinations/salesforce_oauth/db-config.json index 73934db2d..f01b6e89b 100644 --- a/src/configurations/destinations/salesforce_oauth/db-config.json +++ b/src/configurations/destinations/salesforce_oauth/db-config.json @@ -37,5 +37,7 @@ }, "secretKeys": [] }, - "options": {} + "options": { + "hidden": true + } } From f40d5cda403e39e149f132dc1ffc2d1efa50a186 Mon Sep 17 00:00:00 2001 From: George Bardis <109069547+bardisg@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:13:52 +0200 Subject: [PATCH 16/20] feat: add GCM to FB Pixel, GA, GMT, HobSpot, Redis, S3, Slack, VWO (#1190) --- .../facebook_pixel/db-config.json | 17 +- .../destinations/facebook_pixel/schema.json | 511 +++++++++++++++++ .../facebook_pixel/ui-config.json | 209 +++++-- .../destinations/ga/db-config.json | 20 +- .../destinations/ga/schema.json | 511 +++++++++++++++++ .../destinations/ga/ui-config.json | 107 +++- .../destinations/gtm/db-config.json | 7 +- .../destinations/gtm/schema.json | 63 +++ .../destinations/gtm/ui-config.json | 120 +++- .../destinations/hs/db-config.json | 17 +- .../destinations/hs/schema.json | 523 ++++++++++++++++++ .../destinations/hs/ui-config.json | 185 ++++++- .../destinations/redis/db-config.json | 18 +- .../destinations/redis/schema.json | 523 ++++++++++++++++++ .../destinations/redis/ui-config.json | 120 +++- .../destinations/s3/db-config.json | 18 +- .../destinations/s3/schema.json | 523 ++++++++++++++++++ .../destinations/s3/ui-config.json | 120 +++- .../destinations/slack/db-config.json | 18 +- .../destinations/slack/schema.json | 523 ++++++++++++++++++ .../destinations/slack/ui-config.json | 120 +++- .../destinations/vwo/db-config.json | 9 +- .../destinations/vwo/schema.json | 63 +++ .../destinations/vwo/ui-config.json | 120 +++- .../destinations/facebook_pixel.json | 103 ++++ test/data/validation/destinations/ga.json | 98 ++++ test/data/validation/destinations/gtm.json | 98 ++++ test/data/validation/destinations/hs.json | 103 ++++ test/data/validation/destinations/redis.json | 99 ++++ test/data/validation/destinations/s3.json | 98 ++++ test/data/validation/destinations/slack.json | 98 ++++ test/data/validation/destinations/vwo.json | 98 ++++ 32 files changed, 5164 insertions(+), 96 deletions(-) diff --git a/src/configurations/destinations/facebook_pixel/db-config.json b/src/configurations/destinations/facebook_pixel/db-config.json index dd1bfa9d1..e033326ca 100644 --- a/src/configurations/destinations/facebook_pixel/db-config.json +++ b/src/configurations/destinations/facebook_pixel/db-config.json @@ -16,9 +16,10 @@ "blacklistedEvents", "whitelistedEvents", "oneTrustCookieCategories", + "ketchConsentPurposes", + "consentManagement", "eventFilteringOption", - "useUpdatedMapping", - "ketchConsentPurposes" + "useUpdatedMapping" ], "excludeKeys": [], "supportedSourceTypes": [ @@ -73,7 +74,17 @@ "oneTrustCookieCategories", "ketchConsentPurposes" ], - "web": ["useNativeSDK", "legacyConversionPixelId", "connectionMode"] + "web": ["useNativeSDK", "legacyConversionPixelId", "connectionMode", "consentManagement"], + "android": ["consentManagement"], + "ios": ["consentManagement"], + "unity": ["consentManagement"], + "amp": ["consentManagement"], + "cloud": ["consentManagement"], + "warehouse": ["consentManagement"], + "reactnative": ["consentManagement"], + "flutter": ["consentManagement"], + "cordova": ["consentManagement"], + "shopify": ["consentManagement"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/facebook_pixel/schema.json b/src/configurations/destinations/facebook_pixel/schema.json index e9f4fa0d9..db8894d50 100644 --- a/src/configurations/destinations/facebook_pixel/schema.json +++ b/src/configurations/destinations/facebook_pixel/schema.json @@ -214,6 +214,517 @@ } } } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "warehouse": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } } } diff --git a/src/configurations/destinations/facebook_pixel/ui-config.json b/src/configurations/destinations/facebook_pixel/ui-config.json index 9104baa24..80f81c87e 100644 --- a/src/configurations/destinations/facebook_pixel/ui-config.json +++ b/src/configurations/destinations/facebook_pixel/ui-config.json @@ -157,6 +157,90 @@ } ] }, + { + "id": "consentSettings", + "title": "Consent settings", + "note": "Configure consent settings for each provider here", + "icon": "settings", + "groups": [ + { + "title": "OneTrust consent settings", + "note": [ + "Enter your OneTrust consent category IDs if you have them configured. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/onetrust/" + }, + "about RudderStack's OneTrust Consent Management feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Consent categories", + "note": "Input your OneTrust category IDs by pressing 'Enter' after each entry.", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: C0001", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "title": "Ketch consent purpose settings", + "note": [ + "Enter your Ketch purpose Id if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/ketch/" + }, + "about RudderStack and Ketch Consent Manager integration." + ], + "fields": [ + { + "type": "tagInput", + "label": "Purpose ID", + "note": "Input your Ketch purpose Id by pressing 'Enter' after each entry", + "configKey": "ketchConsentPurposes", + "tagKey": "purpose", + "placeholder": "e.g: Marketing", + "default": [ + { + "purpose": "" + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + } + ] + }, { "title": "Other settings", "note": "Configure advanced RudderStack features here", @@ -230,51 +314,6 @@ } } ] - }, - { - "title": "OneTrust consent settings", - "note": [ - "Enter your OneTrust consent category IDs if you have them configured. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", - { - "text": "Learn more ", - "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/onetrust/" - }, - "about RudderStack's OneTrust Consent Management feature." - ], - "fields": [ - { - "type": "tagInput", - "label": "Consent categories", - "note": "Input your OneTrust category IDs by pressing 'Enter' after each entry.", - "configKey": "oneTrustCookieCategories", - "tagKey": "oneTrustCookieCategory", - "placeholder": "e.g: C0001", - "default": [ - { - "oneTrustCookieCategory": "" - } - ] - } - ] - }, - { - "title": "Ketch consent settings", - "note": "Enter your Ketch Consent Purposes if you have them configured.", - "fields": [ - { - "type": "tagInput", - "label": "Ketch consent purpose", - "note": "Input your Ketch consent purpose by pressing 'Enter' after each entry", - "configKey": "ketchConsentPurposes", - "tagKey": "purpose", - "placeholder": "e.g: Marketing", - "default": [ - { - "purpose": "" - } - ] - } - ] } ] } @@ -315,6 +354,86 @@ } ] }, + "consentSettingsTemplate": { + "title": "Consent settings", + "note": "not visible in the ui", + "fields": [ + { + "type": "dynamicCustomForm", + "configKey": "consentManagement", + "default": [], + "rowFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "configKey": "provider", + "options": [ + { + "label": "Custom", + "value": "custom" + }, + { + "label": "Ketch", + "value": "ketch" + }, + { + "label": "OneTrust", + "value": "oneTrust" + } + ], + "default": "oneTrust", + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "configKey": "resolutionStrategy", + "options": [ + { + "label": "AND", + "value": "and" + }, + { + "label": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "tagInput", + "label": "Enter consent category ID’s", + "note": "Input your consent category IDs by pressing ‘Enter’ after each entry. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "configKey": "consents", + "tagKey": "consent", + "placeholder": "e.g: Marketing", + "default": [ + { + "consent": "" + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } + } + ] + }, "redirectGroups": { "customEventMapping": { "tabs": [ diff --git a/src/configurations/destinations/ga/db-config.json b/src/configurations/destinations/ga/db-config.json index 2b871bd7a..1eaec1158 100644 --- a/src/configurations/destinations/ga/db-config.json +++ b/src/configurations/destinations/ga/db-config.json @@ -38,8 +38,9 @@ "blacklistedEvents", "whitelistedEvents", "oneTrustCookieCategories", - "eventFilteringOption", - "ketchConsentPurposes" + "ketchConsentPurposes", + "consentManagement", + "eventFilteringOption" ], "excludeKeys": [], "supportedSourceTypes": [ @@ -109,8 +110,19 @@ "domain", "optimize", "useGoogleAmpClientId", - "namedTracker" - ] + "namedTracker", + "consentManagement" + ], + "android": ["consentManagement"], + "ios": ["consentManagement"], + "unity": ["consentManagement"], + "amp": ["consentManagement"], + "cloud": ["consentManagement"], + "warehouse": ["consentManagement"], + "reactnative": ["consentManagement"], + "flutter": ["consentManagement"], + "cordova": ["consentManagement"], + "shopify": ["consentManagement"] }, "secretKeys": [] }, diff --git a/src/configurations/destinations/ga/schema.json b/src/configurations/destinations/ga/schema.json index a5ebfd333..6d686c239 100644 --- a/src/configurations/destinations/ga/schema.json +++ b/src/configurations/destinations/ga/schema.json @@ -277,6 +277,517 @@ } } } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "warehouse": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } } } diff --git a/src/configurations/destinations/ga/ui-config.json b/src/configurations/destinations/ga/ui-config.json index 6e61a1b0e..befbd7a4e 100644 --- a/src/configurations/destinations/ga/ui-config.json +++ b/src/configurations/destinations/ga/ui-config.json @@ -328,7 +328,19 @@ "label": "Category ID", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } }, { "type": "dynamicCustomForm", @@ -343,7 +355,98 @@ "regex": "^(.{0,100})$", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } } ] } diff --git a/src/configurations/destinations/gtm/db-config.json b/src/configurations/destinations/gtm/db-config.json index 899eee89a..79d921f01 100644 --- a/src/configurations/destinations/gtm/db-config.json +++ b/src/configurations/destinations/gtm/db-config.json @@ -10,6 +10,8 @@ "blacklistedEvents", "whitelistedEvents", "oneTrustCookieCategories", + "ketchConsentPurposes", + "consentManagement", "eventFilteringOption" ], "excludeKeys": [], @@ -29,9 +31,10 @@ "blacklistedEvents", "whitelistedEvents", "eventFilteringOption", - "oneTrustCookieCategories" + "oneTrustCookieCategories", + "ketchConsentPurposes" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "consentManagement"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/gtm/schema.json b/src/configurations/destinations/gtm/schema.json index b368b587c..8a2e3da8a 100644 --- a/src/configurations/destinations/gtm/schema.json +++ b/src/configurations/destinations/gtm/schema.json @@ -60,6 +60,69 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "^(.{0,100})$" + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } } } diff --git a/src/configurations/destinations/gtm/ui-config.json b/src/configurations/destinations/gtm/ui-config.json index 05ab273ba..4a2024459 100644 --- a/src/configurations/destinations/gtm/ui-config.json +++ b/src/configurations/destinations/gtm/ui-config.json @@ -108,7 +108,125 @@ "label": "Category ID", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } } ] } diff --git a/src/configurations/destinations/hs/db-config.json b/src/configurations/destinations/hs/db-config.json index 0379fd7de..bd9d07caa 100644 --- a/src/configurations/destinations/hs/db-config.json +++ b/src/configurations/destinations/hs/db-config.json @@ -11,6 +11,8 @@ "blacklistedEvents", "whitelistedEvents", "oneTrustCookieCategories", + "ketchConsentPurposes", + "consentManagement", "eventFilteringOption" ], "excludeKeys": [], @@ -60,9 +62,20 @@ "whitelistedEvents", "eventFilteringOption", "doAssociation", - "oneTrustCookieCategories" + "oneTrustCookieCategories", + "ketchConsentPurposes" ], - "web": ["useNativeSDK", "connectionMode"] + "web": ["useNativeSDK", "connectionMode", "consentManagement"], + "android": ["consentManagement"], + "ios": ["consentManagement"], + "unity": ["consentManagement"], + "amp": ["consentManagement"], + "cloud": ["consentManagement"], + "warehouse": ["consentManagement"], + "reactnative": ["consentManagement"], + "flutter": ["consentManagement"], + "cordova": ["consentManagement"], + "shopify": ["consentManagement"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/hs/schema.json b/src/configurations/destinations/hs/schema.json index 4b9b61386..deca95dd6 100644 --- a/src/configurations/destinations/hs/schema.json +++ b/src/configurations/destinations/hs/schema.json @@ -66,6 +66,529 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "warehouse": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } }, "allOf": [ diff --git a/src/configurations/destinations/hs/ui-config.json b/src/configurations/destinations/hs/ui-config.json index 2bf2d0ac6..997933543 100644 --- a/src/configurations/destinations/hs/ui-config.json +++ b/src/configurations/destinations/hs/ui-config.json @@ -155,6 +155,90 @@ } ] }, + { + "id": "consentSettings", + "title": "Consent settings", + "note": "Configure consent settings for each provider here", + "icon": "settings", + "groups": [ + { + "title": "OneTrust consent settings", + "note": [ + "Enter your OneTrust consent category IDs if you have them configured. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/onetrust/" + }, + "about RudderStack's OneTrust Consent Management feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Consent categories", + "note": "Input your OneTrust category IDs by pressing 'Enter' after each entry.", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: C0001", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "title": "Ketch consent purpose settings", + "note": [ + "Enter your Ketch purpose Id if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/ketch/" + }, + "about RudderStack and Ketch Consent Manager integration." + ], + "fields": [ + { + "type": "tagInput", + "label": "Purpose ID", + "note": "Input your Ketch purpose Id by pressing 'Enter' after each entry", + "configKey": "ketchConsentPurposes", + "tagKey": "purpose", + "placeholder": "e.g: Marketing", + "default": [ + { + "purpose": "" + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + } + ] + }, { "title": "Other settings", "note": "Configure advanced RudderStack features here", @@ -226,27 +310,6 @@ } } ] - }, - { - "title": "OneTrust consent settings", - "note": [ - "Enter your OneTrust consent category IDs if you have them configured. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", - { - "text": "Learn more", - "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/consent-manager/onetrust/" - }, - " about RudderStack's OneTrust Consent Management feature." - ], - "fields": [ - { - "type": "tagInput", - "label": "Consent categories", - "note": "Input your OneTrust consent category IDs by pressing 'Enter' after each entry.", - "configKey": "oneTrustCookieCategories", - "tagKey": "oneTrustCookieCategory", - "placeholder": "e.g: C0001" - } - ] } ] } @@ -286,6 +349,86 @@ "title": "SDK settings", "fields": [] }, + "consentSettingsTemplate": { + "title": "Consent settings", + "note": "not visible in the ui", + "fields": [ + { + "type": "dynamicCustomForm", + "configKey": "consentManagement", + "default": [], + "rowFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "configKey": "provider", + "options": [ + { + "label": "Custom", + "value": "custom" + }, + { + "label": "Ketch", + "value": "ketch" + }, + { + "label": "OneTrust", + "value": "oneTrust" + } + ], + "default": "oneTrust", + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "configKey": "resolutionStrategy", + "options": [ + { + "label": "AND", + "value": "and" + }, + { + "label": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "tagInput", + "label": "Enter consent category ID’s", + "note": "Input your consent category IDs by pressing ‘Enter’ after each entry. The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "configKey": "consents", + "tagKey": "consent", + "placeholder": "e.g: Marketing", + "default": [ + { + "consent": "" + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } + } + ] + }, "redirectGroups": { "customEventMapping": { "fields": [ diff --git a/src/configurations/destinations/redis/db-config.json b/src/configurations/destinations/redis/db-config.json index f3f6d2325..bb27dee88 100644 --- a/src/configurations/destinations/redis/db-config.json +++ b/src/configurations/destinations/redis/db-config.json @@ -4,7 +4,7 @@ "config": { "transformAtV1": "processor", "saveDestinationResponse": true, - "includeKeys": ["oneTrustCookieCategories"], + "includeKeys": ["oneTrustCookieCategories", "ketchConsentPurposes", "consentManagement"], "excludeKeys": [], "secretKeys": ["password", "caCertificate"], "supportedSourceTypes": [ @@ -44,8 +44,20 @@ "skipVerify", "caCertificate", "clusterMode", - "oneTrustCookieCategories" - ] + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "android": ["consentManagement"], + "ios": ["consentManagement"], + "web": ["consentManagement"], + "unity": ["consentManagement"], + "amp": ["consentManagement"], + "cloud": ["consentManagement"], + "warehouse": ["consentManagement"], + "reactnative": ["consentManagement"], + "flutter": ["consentManagement"], + "cordova": ["consentManagement"], + "shopify": ["consentManagement"] } } } diff --git a/src/configurations/destinations/redis/schema.json b/src/configurations/destinations/redis/schema.json index 5785beec2..d3c0a721d 100644 --- a/src/configurations/destinations/redis/schema.json +++ b/src/configurations/destinations/redis/schema.json @@ -35,6 +35,529 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "^(.{0,100})$" + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "warehouse": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } }, "allOf": [ diff --git a/src/configurations/destinations/redis/ui-config.json b/src/configurations/destinations/redis/ui-config.json index 40ace3e8e..1ca979064 100644 --- a/src/configurations/destinations/redis/ui-config.json +++ b/src/configurations/destinations/redis/ui-config.json @@ -98,7 +98,125 @@ "label": "Category ID", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } } ] } diff --git a/src/configurations/destinations/s3/db-config.json b/src/configurations/destinations/s3/db-config.json index b80380272..4568175f8 100644 --- a/src/configurations/destinations/s3/db-config.json +++ b/src/configurations/destinations/s3/db-config.json @@ -4,7 +4,7 @@ "config": { "transformAtV1": "none", "saveDestinationResponse": true, - "includeKeys": ["oneTrustCookieCategories"], + "includeKeys": ["oneTrustCookieCategories", "ketchConsentPurposes", "consentManagement"], "excludeKeys": [], "supportedSourceTypes": [ "android", @@ -39,8 +39,20 @@ "accessKeyID", "accessKey", "enableSSE", - "oneTrustCookieCategories" - ] + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "android": ["consentManagement"], + "ios": ["consentManagement"], + "web": ["consentManagement"], + "unity": ["consentManagement"], + "amp": ["consentManagement"], + "cloud": ["consentManagement"], + "warehouse": ["consentManagement"], + "reactnative": ["consentManagement"], + "flutter": ["consentManagement"], + "cordova": ["consentManagement"], + "shopify": ["consentManagement"] }, "secretKeys": ["accessKeyID", "accessKey"] } diff --git a/src/configurations/destinations/s3/schema.json b/src/configurations/destinations/s3/schema.json index f79aed80c..aaf665afa 100644 --- a/src/configurations/destinations/s3/schema.json +++ b/src/configurations/destinations/s3/schema.json @@ -31,6 +31,529 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "^(.{0,100})$" + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "warehouse": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } }, "allOf": [ diff --git a/src/configurations/destinations/s3/ui-config.json b/src/configurations/destinations/s3/ui-config.json index e5e021cda..2688e0f89 100644 --- a/src/configurations/destinations/s3/ui-config.json +++ b/src/configurations/destinations/s3/ui-config.json @@ -97,7 +97,125 @@ "label": "Category ID", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } } ] } diff --git a/src/configurations/destinations/slack/db-config.json b/src/configurations/destinations/slack/db-config.json index 391f2acc0..bea738fe7 100644 --- a/src/configurations/destinations/slack/db-config.json +++ b/src/configurations/destinations/slack/db-config.json @@ -4,7 +4,7 @@ "config": { "transformAtV1": "processor", "saveDestinationResponse": true, - "includeKeys": ["oneTrustCookieCategories"], + "includeKeys": ["oneTrustCookieCategories", "ketchConsentPurposes", "consentManagement"], "excludeKeys": [], "supportedSourceTypes": [ "android", @@ -42,8 +42,20 @@ "webhookUrl", "whitelistedTraitsSettings", "denyListOfEvents", - "oneTrustCookieCategories" - ] + "oneTrustCookieCategories", + "ketchConsentPurposes" + ], + "android": ["consentManagement"], + "ios": ["consentManagement"], + "web": ["consentManagement"], + "unity": ["consentManagement"], + "amp": ["consentManagement"], + "cloud": ["consentManagement"], + "warehouse": ["consentManagement"], + "reactnative": ["consentManagement"], + "flutter": ["consentManagement"], + "cordova": ["consentManagement"], + "shopify": ["consentManagement"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/slack/schema.json b/src/configurations/destinations/slack/schema.json index da722d9f7..29a342516 100644 --- a/src/configurations/destinations/slack/schema.json +++ b/src/configurations/destinations/slack/schema.json @@ -96,6 +96,529 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "^(.{0,100})$" + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "android": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "ios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "unity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "amp": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "warehouse": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "reactnative": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "flutter": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "cordova": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + }, + "shopify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } } } diff --git a/src/configurations/destinations/slack/ui-config.json b/src/configurations/destinations/slack/ui-config.json index 3963c494c..39a146a4e 100644 --- a/src/configurations/destinations/slack/ui-config.json +++ b/src/configurations/destinations/slack/ui-config.json @@ -185,7 +185,125 @@ "label": "Category ID", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } } ] } diff --git a/src/configurations/destinations/vwo/db-config.json b/src/configurations/destinations/vwo/db-config.json index 7242a6533..21d0bfa7b 100644 --- a/src/configurations/destinations/vwo/db-config.json +++ b/src/configurations/destinations/vwo/db-config.json @@ -15,7 +15,9 @@ "blacklistedEvents", "whitelistedEvents", "eventFilteringOption", - "oneTrustCookieCategories" + "oneTrustCookieCategories", + "ketchConsentPurposes", + "consentManagement" ], "excludeKeys": [], "supportedSourceTypes": ["web"], @@ -39,9 +41,10 @@ "blacklistedEvents", "whitelistedEvents", "eventFilteringOption", - "oneTrustCookieCategories" + "oneTrustCookieCategories", + "ketchConsentPurposes" ], - "web": ["useNativeSDK"] + "web": ["useNativeSDK", "consentManagement"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/vwo/schema.json b/src/configurations/destinations/vwo/schema.json index 399458491..046bc1e60 100644 --- a/src/configurations/destinations/vwo/schema.json +++ b/src/configurations/destinations/vwo/schema.json @@ -80,6 +80,69 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "^(.{0,100})$" + } + } + } + }, + "consentManagement": { + "type": "object", + "properties": { + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": ["custom", "ketch", "oneTrust"], + "default": "oneTrust" + }, + "consents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "consent": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "provider": { + "const": "custom" + } + }, + "required": ["provider"] + }, + "then": { + "properties": { + "resolutionStrategy": { + "type": "string", + "enum": ["and", "or"] + } + }, + "required": ["resolutionStrategy"] + } + } + ] + } + } + } } } } diff --git a/src/configurations/destinations/vwo/ui-config.json b/src/configurations/destinations/vwo/ui-config.json index cb2eb71f6..f506d5f60 100644 --- a/src/configurations/destinations/vwo/ui-config.json +++ b/src/configurations/destinations/vwo/ui-config.json @@ -146,7 +146,125 @@ "label": "Category ID", "required": false } - ] + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": false + }, + { + "configKey": "AMP_enable-gcm" + } + ], + "featureFlagsCondition": "or" + } + }, + { + "type": "dynamicCustomForm", + "value": "consentManagement", + "label": "Consent management settings", + "footerNote": "The support for category names is deprecated. We recommend using the category IDs instead of the names as IDs are unique and less likely to change over time, making them a more reliable choice.", + "customFields": [ + { + "type": "singleSelect", + "label": "Consent management provider", + "value": "provider", + "options": [ + { + "name": "Custom", + "value": "custom" + }, + { + "name": "Ketch", + "value": "ketch" + }, + { + "name": "OneTrust", + "value": "oneTrust" + } + ], + "defaultOption": { + "name": "OneTrust", + "value": "oneTrust" + }, + "required": true + }, + { + "type": "singleSelect", + "label": "the required consent logic", + "value": "resolutionStrategy", + "options": [ + { + "name": "AND", + "value": "and" + }, + { + "name": "OR", + "value": "or" + } + ], + "required": true, + "variant": "badge", + "preRequisites": { + "fields": [ + { + "configKey": "provider", + "value": "custom" + } + ] + } + }, + { + "type": "dynamicCustomForm", + "value": "consents", + "label": "Enter consent category ID’s", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "consent", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "required": false + } + ] + } + ], + "preRequisites": { + "featureFlags": [ + { + "configKey": "AMP_enable-gcm", + "value": true + } + ] + } } ] } diff --git a/test/data/validation/destinations/facebook_pixel.json b/test/data/validation/destinations/facebook_pixel.json index 979070406..2b5100f3d 100644 --- a/test/data/validation/destinations/facebook_pixel.json +++ b/test/data/validation/destinations/facebook_pixel.json @@ -289,5 +289,108 @@ }, "result": false, "err": ["eventFilteringOption must be equal to one of the allowed values"] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "pixelId": "471828257250906", + "accessToken": "EABAKGVmm4FjBAKAHtWZB4er7fVTjFdJD33daH3ZB340qWk7Nv9MwZCOE0f27EKZA1LQxkQD8hQy1ZBb3SkIgeLcqZAbVtVXbpumoOtpqkP9fRqM0bZCeozeWkH3Y4TZCUpUeUsgKwGDqZB9QrrZCGwRcF9YZAvPDjZAT0YJjWNWQCKhnMQ2OEnrjtJmJgsWDicELxEAZD", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "pixelId": "471828257250906", + "accessToken": "EABAKGVmm4FjBAKAHtWZB4er7fVTjFdJD33daH3ZB340qWk7Nv9MwZCOE0f27EKZA1LQxkQD8hQy1ZBb3SkIgeLcqZAbVtVXbpumoOtpqkP9fRqM0bZCeozeWkH3Y4TZCUpUeUsgKwGDqZB9QrrZCGwRcF9YZAvPDjZAT0YJjWNWQCKhnMQ2OEnrjtJmJgsWDicELxEAZD", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "pixelId": "471828257250906", + "accessToken": "EABAKGVmm4FjBAKAHtWZB4er7fVTjFdJD33daH3ZB340qWk7Nv9MwZCOE0f27EKZA1LQxkQD8hQy1ZBb3SkIgeLcqZAbVtVXbpumoOtpqkP9fRqM0bZCeozeWkH3Y4TZCUpUeUsgKwGDqZB9QrrZCGwRcF9YZAvPDjZAT0YJjWNWQCKhnMQ2OEnrjtJmJgsWDicELxEAZD", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "pixelId": "471828257250906", + "accessToken": "EABAKGVmm4FjBAKAHtWZB4er7fVTjFdJD33daH3ZB340qWk7Nv9MwZCOE0f27EKZA1LQxkQD8hQy1ZBb3SkIgeLcqZAbVtVXbpumoOtpqkP9fRqM0bZCeozeWkH3Y4TZCUpUeUsgKwGDqZB9QrrZCGwRcF9YZAvPDjZAT0YJjWNWQCKhnMQ2OEnrjtJmJgsWDicELxEAZD", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "pixelId": "471828257250906", + "accessToken": "EABAKGVmm4FjBAKAHtWZB4er7fVTjFdJD33daH3ZB340qWk7Nv9MwZCOE0f27EKZA1LQxkQD8hQy1ZBb3SkIgeLcqZAbVtVXbpumoOtpqkP9fRqM0bZCeozeWkH3Y4TZCUpUeUsgKwGDqZB9QrrZCGwRcF9YZAvPDjZAT0YJjWNWQCKhnMQ2OEnrjtJmJgsWDicELxEAZD", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/ga.json b/test/data/validation/destinations/ga.json index f99fd00cf..9e641adb6 100644 --- a/test/data/validation/destinations/ga.json +++ b/test/data/validation/destinations/ga.json @@ -300,5 +300,103 @@ "err": [ "serverSideIdentifyEventAction must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(^(.{0,100})$)\"" ] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "trackingID": "UA-152564043-4", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "trackingID": "UA-152564043-4", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "trackingID": "UA-152564043-4", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "trackingID": "UA-152564043-4", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "trackingID": "UA-152564043-4", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/gtm.json b/test/data/validation/destinations/gtm.json index 4de5490d6..46b229df5 100644 --- a/test/data/validation/destinations/gtm.json +++ b/test/data/validation/destinations/gtm.json @@ -114,5 +114,103 @@ "err": [ "serverUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(?:http(s)?:\\/\\/)?[\\w.-]+(?:\\.[\\w\\.-]+)+[\\w\\-\\._~:/?#[\\]@!\\$&'\\(\\)\\*\\+,;=.]*|^$\"" ] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "containerID": "GTM-ADDA", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "containerID": "GTM-ADDA", + "consentManagement": { + "web": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.web.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.web.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "containerID": "GTM-ADDA", + "consentManagement": { + "web": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.web.0 must have required property 'resolutionStrategy'", + "consentManagement.web.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "containerID": "GTM-ADDA", + "consentManagement": { + "web": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "containerID": "GTM-ADDA", + "consentManagement": { + "web": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.web.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/hs.json b/test/data/validation/destinations/hs.json index 91d6b53d3..273cb1ec1 100644 --- a/test/data/validation/destinations/hs.json +++ b/test/data/validation/destinations/hs.json @@ -198,5 +198,108 @@ ] }, "result": true + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "hubID": "20262117", + "apiKey": "9ege7142-11be-10bc-a168-df1c714326fv", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "hubID": "20262117", + "apiKey": "9ege7142-11be-10bc-a168-df1c714326fv", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "hubID": "20262117", + "apiKey": "9ege7142-11be-10bc-a168-df1c714326fv", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "hubID": "20262117", + "apiKey": "9ege7142-11be-10bc-a168-df1c714326fv", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "hubID": "20262117", + "apiKey": "9ege7142-11be-10bc-a168-df1c714326fv", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/redis.json b/test/data/validation/destinations/redis.json index c18d7036c..f90f743a1 100644 --- a/test/data/validation/destinations/redis.json +++ b/test/data/validation/destinations/redis.json @@ -62,5 +62,104 @@ }, "result": false, "err": [" must have required property 'address'"] + }, + + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "address": "test-elastic-cache.012ut9.ng.0001.use1.cache.amazonaws.com:6379", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "address": "test-elastic-cache.012ut9.ng.0001.use1.cache.amazonaws.com:6379", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "address": "test-elastic-cache.012ut9.ng.0001.use1.cache.amazonaws.com:6379", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "address": "test-elastic-cache.012ut9.ng.0001.use1.cache.amazonaws.com:6379", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "address": "test-elastic-cache.012ut9.ng.0001.use1.cache.amazonaws.com:6379", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/s3.json b/test/data/validation/destinations/s3.json index b152afda2..a937ad6da 100644 --- a/test/data/validation/destinations/s3.json +++ b/test/data/validation/destinations/s3.json @@ -69,5 +69,103 @@ }, "result": false, "err": ["prefix must be string"] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "bucketName": "iqw.btex.deltalake.production", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "bucketName": "iqw.btex.deltalake.production", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "bucketName": "iqw.btex.deltalake.production", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "bucketName": "iqw.btex.deltalake.production", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "bucketName": "iqw.btex.deltalake.production", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/slack.json b/test/data/validation/destinations/slack.json index 097c284f4..4a5850852 100644 --- a/test/data/validation/destinations/slack.json +++ b/test/data/validation/destinations/slack.json @@ -96,5 +96,103 @@ "eventTemplateSettings.0.eventName must be string", "eventTemplateSettings.0.eventRegex must be boolean" ] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "webhookUrl": "https://hooks.slack.com/services/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", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "webhookUrl": "https://hooks.slack.com/services/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", + "consentManagement": { + "android": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "webhookUrl": "https://hooks.slack.com/services/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", + "consentManagement": { + "android": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.android.0 must have required property 'resolutionStrategy'", + "consentManagement.android.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "webhookUrl": "https://hooks.slack.com/services/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", + "consentManagement": { + "android": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "webhookUrl": "https://hooks.slack.com/services/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", + "consentManagement": { + "android": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.android.0.provider must be equal to one of the allowed values"] } ] diff --git a/test/data/validation/destinations/vwo.json b/test/data/validation/destinations/vwo.json index 15c9f2666..945844b57 100644 --- a/test/data/validation/destinations/vwo.json +++ b/test/data/validation/destinations/vwo.json @@ -105,5 +105,103 @@ }, "result": false, "err": ["whitelistedEvents.0.eventName must be string"] + }, + { + "testTitle": "With valid multiple consent management providers config", + "config": { + "accountId": "68869", + "consentManagement": { + "web": [ + { + "provider": "custom", + "consents": [ + { + "consent": "Marketing" + } + ], + "resolutionStrategy": "or" + }, + { + "provider": "oneTrust", + "consents": [ + { + "consent": "Marketing" + } + ] + }, + { + "provider": "ketch", + "consents": [] + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config and invalid resolutionStrategy value", + "config": { + "accountId": "68869", + "consentManagement": { + "web": [ + { + "provider": "custom", + "resolutionStrategy": "nor" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.web.0.resolutionStrategy must be equal to one of the allowed values", + "consentManagement.web.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management custom provider config and no resolutionStrategy value", + "config": { + "accountId": "68869", + "consentManagement": { + "web": [ + { + "provider": "custom" + } + ] + } + }, + "result": false, + "err": [ + "consentManagement.web.0 must have required property 'resolutionStrategy'", + "consentManagement.web.0 must match \"then\" schema" + ] + }, + { + "testTitle": "With consent management OneTrust provider config and no resolutionStrategy value", + "config": { + "accountId": "68869", + "consentManagement": { + "web": [ + { + "provider": "oneTrust" + } + ] + } + }, + "result": true + }, + { + "testTitle": "With consent management custom provider config invalid provider value", + "config": { + "accountId": "68869", + "consentManagement": { + "web": [ + { + "provider": "dummyProvider" + } + ] + } + }, + "result": false, + "err": ["consentManagement.web.0.provider must be equal to one of the allowed values"] } ] From 09bbe2a5d1f81c26dec71582fbb7069550e7c7af Mon Sep 17 00:00:00 2001 From: Ruchira Moitra <31311577+ruchiramoitra@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:20:47 +0530 Subject: [PATCH 17/20] feat: enabled sql model support for mysql (#1222) --- src/configurations/sources/mysql/db-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configurations/sources/mysql/db-config.json b/src/configurations/sources/mysql/db-config.json index 0dac79fd6..ebc893105 100644 --- a/src/configurations/sources/mysql/db-config.json +++ b/src/configurations/sources/mysql/db-config.json @@ -4,7 +4,7 @@ "displayName": "MySql", "options": { "syncBehaviours": ["upsert", "mirror"], - "isSqlModelSupported": false + "isSqlModelSupported": true }, "type": "warehouse" } From 5e533ba8c4d1aa99f6b601a9aa8d3ecf44e533fc Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:27:48 +0530 Subject: [PATCH 18/20] fix: bluecore review comments addressed (#1225) --- src/configurations/destinations/bluecore/db-config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/configurations/destinations/bluecore/db-config.json b/src/configurations/destinations/bluecore/db-config.json index 2d2b41218..3bd5d144e 100644 --- a/src/configurations/destinations/bluecore/db-config.json +++ b/src/configurations/destinations/bluecore/db-config.json @@ -5,7 +5,6 @@ "cdkV2Enabled": true, "transformAtV1": "processor", "saveDestinationResponse": true, - "includeKeys": ["oneTrustCookieCategories"], "excludeKeys": [], "supportedSourceTypes": [ "android", @@ -40,5 +39,8 @@ "defaultConfig": ["bluecoreNamespace", "eventsMapping", "oneTrustCookieCategories"] }, "secretKeys": ["bluecoreNamespace"] + }, + "options": { + "isBeta": true } } From c9d3ace8ed3dc804932b5e24182e2a32ca51c1d5 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 20 Feb 2024 07:01:24 +0000 Subject: [PATCH 19/20] chore(release): 1.65.0 --- CHANGELOG.md | 23 +++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b61e273..ab8d58f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.65.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.2...v1.65.0) (2024-02-20) + + +### Features + +* add connectionMode for all sourceTypes in dest def for selected destinations ([#1197](https://github.com/rudderlabs/rudder-config-schema/issues/1197)) ([dde686c](https://github.com/rudderlabs/rudder-config-schema/commit/dde686cc72e0ac7423487d566e8f9715cc43d24f)) +* add GCM to FB Pixel, GA, GMT, HobSpot, Redis, S3, Slack, VWO ([#1190](https://github.com/rudderlabs/rudder-config-schema/issues/1190)) ([f40d5cd](https://github.com/rudderlabs/rudder-config-schema/commit/f40d5cda403e39e149f132dc1ffc2d1efa50a186)) +* adding custom field support for freshsales ([#1195](https://github.com/rudderlabs/rudder-config-schema/issues/1195)) ([9124fb4](https://github.com/rudderlabs/rudder-config-schema/commit/9124fb44fd18d903bbf37f7cef3eaf7aeccf92e4)) +* enabled sql model support for mysql ([#1222](https://github.com/rudderlabs/rudder-config-schema/issues/1222)) ([09bbe2a](https://github.com/rudderlabs/rudder-config-schema/commit/09bbe2a5d1f81c26dec71582fbb7069550e7c7af)) +* improve schema generator output ([#1191](https://github.com/rudderlabs/rudder-config-schema/issues/1191)) ([04e05b8](https://github.com/rudderlabs/rudder-config-schema/commit/04e05b89a7333291792fbb6e111388b5ff8b985f)) +* onboard new destination commandbar ([#1196](https://github.com/rudderlabs/rudder-config-schema/issues/1196)) ([daf9fa8](https://github.com/rudderlabs/rudder-config-schema/commit/daf9fa8256eed280bad04c0c109de3787e6262e0)) +* onboard trade desk real time conversions ([#1213](https://github.com/rudderlabs/rudder-config-schema/issues/1213)) ([0d9b8d7](https://github.com/rudderlabs/rudder-config-schema/commit/0d9b8d7ee45b6ba5c914aaca6c2c11474a7575e9)) +* onboarding bluecore integration ([#1182](https://github.com/rudderlabs/rudder-config-schema/issues/1182)) ([6d5e40f](https://github.com/rudderlabs/rudder-config-schema/commit/6d5e40fdb812aa4c46ea13cbae5f7eb696d0cc0c)) +* tiktok_offline_events added support for all Standard events ([#1216](https://github.com/rudderlabs/rudder-config-schema/issues/1216)) ([61d36fa](https://github.com/rudderlabs/rudder-config-schema/commit/61d36fa5db882f00d97dd9dc88b2f26a7d49a87a)) + + +### Bug Fixes + +* add support of placing properties at root in af ([#1203](https://github.com/rudderlabs/rudder-config-schema/issues/1203)) ([1859354](https://github.com/rudderlabs/rudder-config-schema/commit/185935467899222b8766c63c63b934d70410cc9a)) +* bluecore review comments addressed ([#1225](https://github.com/rudderlabs/rudder-config-schema/issues/1225)) ([5e533ba](https://github.com/rudderlabs/rudder-config-schema/commit/5e533ba8c4d1aa99f6b601a9aa8d3ecf44e533fc)) +* comscore schema ([#1217](https://github.com/rudderlabs/rudder-config-schema/issues/1217)) ([7faa64a](https://github.com/rudderlabs/rudder-config-schema/commit/7faa64acfe2fca20b75789f31b541dd33e7fde8c)) +* reverting unhide of salesforce oauth ([#1220](https://github.com/rudderlabs/rudder-config-schema/issues/1220)) ([e73981f](https://github.com/rudderlabs/rudder-config-schema/commit/e73981f54b757aa60181942a19e7db97aac40658)) + ### [1.64.2](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.1...v1.64.2) (2024-02-14) ### [1.64.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.0...v1.64.1) (2024-02-13) diff --git a/package-lock.json b/package-lock.json index aa3bd6dd4..7eb48b94d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-config-schema", - "version": "1.64.2", + "version": "1.65.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-config-schema", - "version": "1.64.2", + "version": "1.65.0", "license": "MIT", "dependencies": { "ajv": "^8.12.0", diff --git a/package.json b/package.json index 9425018ed..a5d73ac97 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-config-schema", - "version": "1.64.2", + "version": "1.65.0", "description": "", "main": "src/index.ts", "private": true, From c0a3c1be79dc15a53a3a49588a9c65cc3bd92328 Mon Sep 17 00:00:00 2001 From: shrouti1507 Date: Tue, 20 Feb 2024 12:49:16 +0530 Subject: [PATCH 20/20] fix: clear up change log --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab8d58f5d..49c44291f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ All notable changes to this project will be documented in this file. See [standa * add connectionMode for all sourceTypes in dest def for selected destinations ([#1197](https://github.com/rudderlabs/rudder-config-schema/issues/1197)) ([dde686c](https://github.com/rudderlabs/rudder-config-schema/commit/dde686cc72e0ac7423487d566e8f9715cc43d24f)) * add GCM to FB Pixel, GA, GMT, HobSpot, Redis, S3, Slack, VWO ([#1190](https://github.com/rudderlabs/rudder-config-schema/issues/1190)) ([f40d5cd](https://github.com/rudderlabs/rudder-config-schema/commit/f40d5cda403e39e149f132dc1ffc2d1efa50a186)) -* adding custom field support for freshsales ([#1195](https://github.com/rudderlabs/rudder-config-schema/issues/1195)) ([9124fb4](https://github.com/rudderlabs/rudder-config-schema/commit/9124fb44fd18d903bbf37f7cef3eaf7aeccf92e4)) * enabled sql model support for mysql ([#1222](https://github.com/rudderlabs/rudder-config-schema/issues/1222)) ([09bbe2a](https://github.com/rudderlabs/rudder-config-schema/commit/09bbe2a5d1f81c26dec71582fbb7069550e7c7af)) * improve schema generator output ([#1191](https://github.com/rudderlabs/rudder-config-schema/issues/1191)) ([04e05b8](https://github.com/rudderlabs/rudder-config-schema/commit/04e05b89a7333291792fbb6e111388b5ff8b985f)) * onboard new destination commandbar ([#1196](https://github.com/rudderlabs/rudder-config-schema/issues/1196)) ([daf9fa8](https://github.com/rudderlabs/rudder-config-schema/commit/daf9fa8256eed280bad04c0c109de3787e6262e0)) @@ -21,9 +20,7 @@ All notable changes to this project will be documented in this file. See [standa ### Bug Fixes * add support of placing properties at root in af ([#1203](https://github.com/rudderlabs/rudder-config-schema/issues/1203)) ([1859354](https://github.com/rudderlabs/rudder-config-schema/commit/185935467899222b8766c63c63b934d70410cc9a)) -* bluecore review comments addressed ([#1225](https://github.com/rudderlabs/rudder-config-schema/issues/1225)) ([5e533ba](https://github.com/rudderlabs/rudder-config-schema/commit/5e533ba8c4d1aa99f6b601a9aa8d3ecf44e533fc)) * comscore schema ([#1217](https://github.com/rudderlabs/rudder-config-schema/issues/1217)) ([7faa64a](https://github.com/rudderlabs/rudder-config-schema/commit/7faa64acfe2fca20b75789f31b541dd33e7fde8c)) -* reverting unhide of salesforce oauth ([#1220](https://github.com/rudderlabs/rudder-config-schema/issues/1220)) ([e73981f](https://github.com/rudderlabs/rudder-config-schema/commit/e73981f54b757aa60181942a19e7db97aac40658)) ### [1.64.2](https://github.com/rudderlabs/rudder-config-schema/compare/v1.64.1...v1.64.2) (2024-02-14)