From 2c1489d42da0e3b8ba41f009ce866f2642922127 Mon Sep 17 00:00:00 2001 From: wnijmeijer Date: Thu, 23 Jul 2020 09:47:08 +0200 Subject: [PATCH] added v2 --- client/cloud_v2.py | 3 + migrate_v2_to_v2.py | 1659 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1662 insertions(+) create mode 100644 migrate_v2_to_v2.py diff --git a/client/cloud_v2.py b/client/cloud_v2.py index 9acdee7..acc85d6 100644 --- a/client/cloud_v2.py +++ b/client/cloud_v2.py @@ -96,6 +96,9 @@ def dimension_create(self, name, event, fields: dict): #https://platform.cloud.coveo.com/rest/ua/v15/dimensions/custom?org=asdf&name=myname&event=searches&updatePastEvents=false return self.do_post(f'rest/ua/v15/dimensions/custom?org={self.org_id}&name={name}{event}', fields) + def dimensions_get(self): + return self.do_get(f'rest/ua/v15/dimensions/?org={self.org_id}&includeOnlyParents=true') + def pipeline_create(self, fields: dict): return self.do_post(f'rest/search/v1/admin/pipelines/?organizationId={self.org_id}', fields) diff --git a/migrate_v2_to_v2.py b/migrate_v2_to_v2.py new file mode 100644 index 0000000..6474545 --- /dev/null +++ b/migrate_v2_to_v2.py @@ -0,0 +1,1659 @@ +#! /usr/bin/python3 +""" +Migrate configuration from CloudV2 to CloudV2. + How to check if something already exists in target? Using Name or ID? + Is there not a script already? --> by PS? +Supported operations: + Move source, + Does it exists? Create else update + including schedules, + field mappings, + if there is an IPE configured, move that too, + if there is a security provider associated, move that too + Move IPE + Move Conditions + Move Query Pipeline + including all statements --> how to check if one exists already? Use Id? + Move Custom Dimensions + Move Analytic Reports +""" + +import json +import itertools +import re +from client.cloud_v2 import * +from client.fields import Fields +import argparse +import jiphy +import os + + +def v1_get_source_id(sources, source_name): + """ Get a source id from a source name (CloudV1) + >>> v1_get_source_id([{'id': '0', 'name': 'FOO'}, {'id': '1', 'name': 'BAR'}], 'bar') + '1' + >>> v1_get_source_id([{'id': '0', 'name': 'FOO'}, {'id': '1', 'name': 'BAR'}], 'new') + Traceback (most recent call last): + ... + ValueError: Source new does not exist + >>> v1_get_source_id([{'id': '0', 'name': 'FOO'}, {'id': '1', 'name': 'FOO'}], 'foo') + Traceback (most recent call last): + ... + ValueError: More than one source foo found. This should not happen. + """ + source_ids = [source['id'] for source in sources + if source['name'].lower() == source_name.lower()] + if not source_ids: + raise ValueError(f'Source {source_name} does not exist') + if len(source_ids) > 1: + raise ValueError(f'More than one source {source_name} found. This should not happen.') + return source_ids[0] + +def v1_get_fields_by_name(fields): + """ Get a dictionary of fields by name (field name -> field) (CloudV1) + >>> v1_get_fields_by_name(( \ + {'id': '0', 'name': 'FOO'}, \ + {'id': '1', 'name': 'bAr'}, \ + {'id': '2', 'name': 'foobar'})) + {'foo': {'id': '0', 'name': 'FOO'}, 'bar': {'id': '1', 'name': 'bAr'}, 'foobar': {'id': '2', 'name': 'foobar'}} + """ + return dict([(f['name'].lower(), f) for f in fields]) + + +def v2_get_fields_in_use(fields, mappings): + """ Get a dictionary of fields that are used in the mappings provided (field name -> field) (CloudV2) + >>> v2_get_fields_in_use(({'items': ({'name': 'FOO'}, {'name': 'bar'}, {'name': 'foobar'})}), ('foo', 'BAR')) + {'foo': {'name': 'FOO'}, 'bar': {'name': 'bar'}} + """ + return dict((field['name'].lower(), field) for field in + fields['items'] if field['name'].lower() in + [mapping.lower() for mapping in mappings]) + + +def v2_get_mappings_fieldname(mappings): + """ Get a dictionary of mappings (field name -> field) (CloudV2) + >>> v2_get_mappings_fieldname({'common': {'rules': []}}) + [] + >>> v2_get_mappings_fieldname({'common': {'rules': [{'field': 'FOO'}, {'field': 'bAr'}]}}) + ['foo', 'bar'] + >>> v2_get_mappings_fieldname({'types': []}) + [] + >>> v2_get_mappings_fieldname({'types': [{'rules': []}]}) + [] + >>> v2_get_mappings_fieldname({'types': [ \ + {'rules': [{'field': 'foo0'}, {'field': 'fOO1'}]}, \ + {'rules': [{'field': 'bar0'}]}]}) + ['foo0', 'foo1', 'bar0'] + >>> v2_get_mappings_fieldname({'common': {'rules': [{'field': 'FOO'}, {'field': 'bAr'}]}, \ + 'types': [ \ + {'rules': [{'field': 'foo0'}, {'field': 'fOO1'}]}, \ + {'rules': [{'field': 'bar0'}]}]}) + ['foo', 'bar', 'foo0', 'foo1', 'bar0'] + """ + # get common mappings + common_mappings = mappings['common']['rules'] if 'common' in mappings and 'rules' in mappings['common'] else [] + # get mytype specifc mappings + type_mappings = list() + type_rules = [rule['rules'] if 'rules' in rule else [] for rule in mappings['types']] if 'types' in mappings else [] + [[type_mappings.append(field) for field in type_rule] for type_rule in type_rules] + # all v2 mappings + v2_all_mappings = common_mappings + type_mappings + return [mapping['field'].lower() for mapping in v2_all_mappings] + + +def get_fields_difference(v1_field, v2_field): + """ Get the differences between 2 fields + >>> get_fields_difference({}, {}) + [] + >>> v1_field = {'sort': True} + >>> v2_field = {'sort': True} + >>> get_fields_difference(v1_field, v2_field) + [] + >>> v1_field = {'fieldQueries': True} + >>> v2_field = {'includeInQuery': False} + >>> get_fields_difference(v1_field, v2_field) + [{'v1ConfigName': 'fieldQueries', 'v2ConfigName': 'includeInQuery', 'v1Value': True, 'v2Value': False}] + >>> v1_field = {'fieldQueries': True, 'sort': False} + >>> v2_field = {'includeInQuery': False, 'sort': True} + >>> get_fields_difference(v1_field, v2_field) + [{'v1ConfigName': 'fieldQueries', 'v2ConfigName': 'includeInQuery', 'v1Value': True, 'v2Value': False}, {'v1ConfigName': 'sort', 'v2ConfigName': 'sort', 'v1Value': False, 'v2Value': True}] + """ + field_diff = [{KEY_V1_CONFIGNAME: diff, + KEY_V2_CONFIGNAME: FIELDS_KEYS_V1_V2[diff], + KEY_V1_VALUE: v1_field[diff], + KEY_V2_VALUE: v2_field[FIELDS_KEYS_V1_V2[diff]]} + for diff in FIELDS_KEYS_V1_V2 + if diff in v1_field and FIELDS_KEYS_V1_V2[diff] in v2_field and v1_field[diff] != v2_field[FIELDS_KEYS_V1_V2[diff]]] + return field_diff + + +def get_fields_differences(v1_fields, v2_fields): + """ Get a list of differences between 2 fields list + >>> get_fields_differences({}, {}) + [] + >>> v1_fields = {'field0': {'sort': True, 'fieldType':'STRING', 'facet': False, 'multivalueFacet': False}, 'field1': {'sort': True, 'fieldType':'STRING', 'facet': True, 'multivalueFacet': False}} + >>> v2_fields = {'field1': {'sort': False, 'facet': False, 'type':'STRING', 'multiValueFacet': False}, 'field2': {'sort': True, 'facet': True, 'type':'STRING', 'multiValueFacet': False }} + >>> get_fields_differences(v1_fields, v2_fields) + [('field1', {'v1Field': {'sort': True, 'fieldType': 'STRING', 'facet': True, 'multivalueFacet': False}, 'v2Field': {'sort': False, 'facet': False, 'type': 'STRING', 'multiValueFacet': False}, 'configDiff': [{'v1ConfigName': 'facet', 'v2ConfigName': 'facet', 'v1Value': True, 'v2Value': False}, {'v1ConfigName': 'sort', 'v2ConfigName': 'sort', 'v1Value': True, 'v2Value': False}]})] + """ + v1_field_names = v1_fields.keys() + v2_field_names = v2_fields.keys() + diffs = list() + for v1_field_name in v1_field_names: + if v1_field_name in v2_field_names: + v1_field = v1_fields[v1_field_name] + #Fix v1_field facet and Multifacet + if v1_field['facet'] and v1_field['multivalueFacet']: + v1_field['facet'] = False + #Fix sorting + mysort = v1_field['sort'] + if v1_field['fieldType']=='DATE' or v1_field['fieldType']=='INTEGER' or v1_field['fieldType']=='DOUBLE' or v1_field['fieldType']=='LONG_64': + mysort = True + v1_field['sort']=mysort + myfieldtype = v1_field['fieldType'] + #fix field types + if myfieldtype=='INTEGER': + myfieldtype='LONG' + v1_field['fieldType']=myfieldtype + v2_field = v2_fields[v1_field_name] + diff = get_fields_difference(v1_field, v2_fields[v1_field_name]) + if diff: + diffs.append((v1_field_name, {KEY_V1_FIELD: v1_field, KEY_V2_FIELD: v2_field, KEY_CONFIG_DIFF: diff})) + return diffs + + +def v2_get_updated_field(field_difference): + global finalreport + """ Get a modified field according to the differences provided (CloudV2) + >>> diffs = ('field1', {'v1Field': {'sort': True, 'facet': True}, 'v2Field': {'name': 'field1', 'sort': False, 'facet': False, 'type': 'SOMEtype'}, 'configDiff': [{'v1ConfigName': 'facet', 'v2ConfigName': 'facet', 'v1Value': True, 'v2Value': False}, {'v1ConfigName': 'sort', 'v2ConfigName': 'sort', 'v1Value': True, 'v2Value': False}]}) + >>> v2_get_updated_field(diffs) + {'name': 'field1', 'sort': True, 'facet': True, 'type': 'SOMEtype'} + """ + for diff in field_difference[1][KEY_CONFIG_DIFF]: + if field_difference[1][KEY_V2_FIELD]['type'].lower() == 'date' and \ + diff[KEY_V2_CONFIGNAME].lower() == 'sort' and \ + diff[KEY_V1_VALUE] == False: + print(f'\t-> Field "{field_difference[1][KEY_V2_FIELD]["name"]}" is of type date and cannot be set to sort = false. Skipping this change.') + finalreport += f'\n\t-> Field "{field_difference[1][KEY_V2_FIELD]["name"]}" is of type date and cannot be set to sort = false. Skipping this change.' + else: + field_difference[1][KEY_V2_FIELD][diff[KEY_V2_CONFIGNAME]] = diff[KEY_V1_VALUE] + return field_difference[1][KEY_V2_FIELD] + + +def v2_get_updated_fields(field_differences): + """ Get a list of modified fields according to the differences provided (CloudV2) + """ + return [v2_get_updated_field(diff) for diff in field_differences] + + +def get_unused_fields(fields): + global finalreport + unused_fields = [] + for item in fields['items']: + if not item['sources'] and not item['system']: + unused_fields.append(item['name']) + print(f'\t-> Field "{item["name"]}" is unused') + finalreport += f'\n\t-> Field "{item["name"]}" is unused' + return ",".join(unused_fields) + + +def v1_get_unique_fields(fields: list) -> list: + """ + # >>> fields = [{'name': 'f0', 'fieldType': 'STRING', 'contentType': 'METADATA', 'sort': True, 'facet': False}, \ + # {'name': 'f0', 'fieldType': 'STRING', 'contentType': 'METADATA', 'sort': False, 'facet': True}, \ + # {'name': 'f1', 'fieldType': 'STRING', 'contentType': 'METADATA'}] + # >>> v1_get_unique_fields(fields) + # [('f0', [{'name': 'f0', 'fieldType': 'STRING', 'contentType': 'METADATA', 'sort': True, 'facet': False}, {'name': 'f0', 'fieldType': 'STRING', 'contentType': 'METADATA', 'sort': False, 'facet': True}]), ('f1', [{'name': 'f1', 'fieldType': 'STRING', 'contentType': 'METADATA'}])] + """ + def get_fields_by_name() -> dict: + fields_by_name = dict() + for field in fields: + name = field['name'] + if name in fields_by_name: + fields_by_name[name].append(field) + else: + fields_by_name[name] = [field] + return fields_by_name + + def merge_fields_config(fields: list) -> dict: + new_field = {'fieldQueries': False, 'freeTextQueries': False, 'facet': False, 'multivalueFacet': False, 'sort': False, 'displayField': False} + mappings = list() + for field in fields: + #print (field) + if (field['contentType']=='SCRIPT'): + mapping = {'name': field['name'], 'contentType':field['contentType'], 'metadataName': field['scriptParams'], 'sourceId': field['sourceId']} + else: + mapping = {'name': field['name'], 'contentType':field['contentType'], 'metadataName': field['metadataName'], 'sourceId': field['sourceId']} + for flag in new_field: + new_field[flag] |= field[flag] + mappings.append(mapping) + new_field['mappings'] = mappings + return new_field + + def validate_field_config(fields: list) -> bool: + expected_length = len(fields) + field_type = fields[0]['fieldType'] + content_type = fields[0]['contentType'] + if expected_length != len(list(filter(lambda f: f['fieldType'] == field_type and + f['contentType'] == content_type, fields))): + return False + return True + + fields_by_name = get_fields_by_name() + unique_fields_by_name = list() + for field_name in fields_by_name: + field = fields_by_name[field_name] + if validate_field_config(field): + merged = merge_fields_config(field) + merged['name'] = field_name + merged['fieldType'] = field[0]['fieldType'] + merged['contentType'] = field[0]['contentType'] + unique_fields_by_name.append((field_name, merged)) + return unique_fields_by_name + + +def v1_field_is_user(field: dict) -> bool: + return field['fieldOrigin'] == 'CUSTOM' and not field['contentType'] == 'CUSTOM_SCRIPT' + #return field['contentType'] == 'CUSTOM' + + +def copy_user_fields(v1_fields: list, v2_client: CloudV2, dry_run: bool): + global finalreport + v2_unique_fields = [Fields.v1_to_v2(field[1]) for field in v1_fields] + v2_fields = [f['name'] for f in v2_client.fields_get()['items']] + v2_fields_to_create = list() + for field in v2_unique_fields: + if field['name'] in v2_fields: + print(f'SKIPPING FIELD \'{field["name"]}\' because it already exists in org: {field}') + finalreport += f'\n\tSKIPPING FIELD \'{field["name"]}\' because it already exists in org: {field}' + else: + print(f'ADDING FIELD \'{field["name"]}\': {field}') + finalreport += f'\n\tADDING FIELD \'{field["name"]}\': {field}' + v2_fields_to_create.append(field) + + if not dry_run and len(v2_fields_to_create) > 0: + v2_client.fields_create_batch(v2_fields_to_create) + +def v2_create_mapping_from_v1_fields(v2_client: CloudV2, v1_sources: object, v1_fields: list, v2_sources: list, dry_run: bool): + def v2_get_mappings_by_source_id_by_field_name(sources: dict) -> dict: + # we can't have the same mapping on the same field in CloudV1 + return dict([(sources[source]['v2_id'], + dict([(x['field'].lower(), x) for x in + v2_client.mappings_get(sources[source]['v2_id'])['common']['rules']])) + for source in sources]) + + def v2_get_sources_by_name() -> dict: + v1_sources_by_name = dict([(source['name'].lower(), source) for source in v1_sources['sources']]) + v2_sources_by_name = dict([(source['name'].lower(), source) for source in v2_sources]) + return dict([(v2_source_key, + {'v1_id': v1_sources_by_name[v2_source_key]['id'], + 'v2_id': v2_sources_by_name[v2_source_key]['id']}) + for v2_source_key in v2_sources_by_name.keys() + if v2_source_key in v1_sources_by_name]) + + def v2_get_source_used_field(field: dict, common_sources: dict) -> dict: + # v1 source id -> v1 source name == v2 source name -> v2 source id + v1_source_id = field['sourceId'] + v1_source_name = v1_sources_by_id[v1_source_id]['name'] + v2_source_id = None + if v1_source_name.lower() in common_sources: + v2_source_id = common_sources[v1_source_name.lower()]['v2_id'] + return {'id': v2_source_id, 'name': v1_source_name} + + def v2_create_mapping(field: dict, mappings: dict, source_id: str, source_name: str) -> None: + global finalreport + #print (field) + if (field['contentType']=='SCRIPT'): + #get content from scriptparams + scriptcontent=field["metadataName"] + new_mapping = {'content': [f'{scriptcontent["Content"]}'], 'field': f'{field["name"]}'} + else: + new_mapping = {'content': [f'%[{field["metadataName"]}]'], 'field': f'{field["name"]}'} + new_mapping_exists = new_mapping['field'].lower() in mappings + if new_mapping_exists: + print(f'SKIPPING MAPPING {new_mapping} because it\'s already present in source \'{source_name}\'') + finalreport += f'\n\tSKIPPING MAPPING {new_mapping} because it\'s already present in source \'{source_name}\'' + elif not dry_run: + print(f'ADD MAPPING: {new_mapping}') + finalreport += f'\n\tADD MAPPING: {new_mapping}' + v2_client.mappings_common_add(source_id, False, new_mapping) + + global finalreport + common_sources = v2_get_sources_by_name() + if len(common_sources) == 0: + print(f'No common source names between CloudV1 and CloudV2. Cannot copy mappings.') + finalreport += f'\n\tNo common source names between CloudV1 and CloudV2. Cannot copy mappings.' + else: + print ("Common source names found.") + #print(f'Common source names ({len(common_sources)}): {json.dumps(common_sources)}') + mappings_by_source_id = v2_get_mappings_by_source_id_by_field_name(common_sources) + v1_sources_by_id = dict([(source['id'].lower(), source) for source in v1_sources['sources']]) + for field in v1_fields: + v2_source = v2_get_source_used_field(field, common_sources) + v2_source_id = v2_source['id'] + v2_source_name = v2_source['name'] + if v2_source_id is None: + print(f'SKIPPING MAPPING for \'{field["name"]}\' because source \'{v2_source_name}\' does not exist in CloudV2') + finalreport += f'\n\tSKIPPING MAPPING for \'{field["name"]}\' because source \'{v2_source_name}\' does not exist in CloudV2' + else: + v2_create_mapping(field, mappings_by_source_id[v2_source_id], v2_source_id, v2_source_name) + +def addLine(): + return " --------------------------------------------------\n" + + +def translatetype(mytype): + fromRepo = { "WEB":"WEB2", "EXCHANGE_ENTERPRISE":"EXCHANGE", + "JIRA":"JIRA2_HOSTED", "JIRA_CLOUD":"JIRA2", "JIVE":"JIVE_HOSTED", "JIVE_CLOUD":"JIVE", + "CONFLUENCE2":"CONFLUENCE2_HOSTED","CONFLUENCE2_CLOUD":"CONFLUENCE2","SHAREPOINT":"SHAREPOINT", + "GMAIL":"GMAIL_SINGLE_USER", + "SHAREPOINT_ONLINE":"SHAREPOINT_ONLINE2","TWITTER":"TWITTER2", + "YAMMER":"BAD","ORACLE_KNOWLEDGE":"BAD","CONFLUENCE":"BAD", + "WEBSCRAPER": "MAN", + #Must be done manual, you need an valid OAUTH token + "SALESFORCE":"SALESFORCE", "KNOWLEDGEBASE":"SALESFORCE","SALESFORCE_CONTENT":"SALESFORCE", + } + + for repo in fromRepo: + if repo == mytype: + return fromRepo[repo] + return mytype + +def addAddressPattern(mytype): + mytype["configuration"]["addressPatterns"]=[] + mytype["configuration"]["addressPatterns"].append({}) + mytype["configuration"]["addressPatterns"][0]["expression"]="*" + mytype["configuration"]["addressPatterns"][0]["patterntype"]="Wildcard" + mytype["configuration"]["addressPatterns"][0]["allowed"]=True + + return mytype + +def addSecurityEverybody(mytype): + mytype["configuration"]["permissions"]=[{"permissionSets": [{"allowedPermissions": [{"identitytype": "Group","securityProvider": "Email Security Provider","identity": "*@*"}],"name": "Shared"}],"name": "Source Specified Permissions"} ] + + return mytype + +def addSecurityUser(mytype,email): + mytype["configuration"]["permissions"]=[{"permissionSets": [{"allowedPermissions": [{"identitytype": "User","securityProvider": "Email Security Provider","identity": email}],"name": "Private"}],"name": "Source Specified Permissions"} ] + + return mytype + + +def addParam(mytype,key,sensitive, value): + mytype["configuration"]["parameters"][key]={} + mytype["configuration"]["parameters"][key]["sensitive"]=sensitive + mytype["configuration"]["parameters"][key]["value"]=value + return mytype + +def getCloudSetting(config, value, default): + if value in config["cloudSourceConfiguration"]: + return config["cloudSourceConfiguration"][value] + else: + return default + +def fix(mytype, config): + global v1_org_id + global actionlist + mytype["MethodToUse"]="SIMPLE" + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="GOOGLE_DRIVE_SINGLE_USER": + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.GoogleDriveSingleUser" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ClientSecret", True, "UPDATEIT") + mytype=addParam(mytype,"ClientId", False, "UPDATEIT") + mytype=addParam(mytype,"ClientRefreshToken", True, "UPDATEIT") + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + email = getCloudSetting(config,"name","").replace('Drive - ','') + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"owner","*@*")) + if email: + mytype["emailAddress"]=email + mytype["additionalInfos"]={"emailAddress": email } + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="GMAIL": + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + #Must be done manual, you need an valid OAUTH token + #Must be done using RAW + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.GmailSingleUser" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ClientSecret", True, "UPDATEIT") + mytype=addParam(mytype,"ClientId", False, "UPDATEIT") + mytype=addParam(mytype,"ClientRefreshToken", True, "UPDATEIT") + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"emailAddress","*@*")) + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["emailAddress"] } + mytype["configuration"]["startingAddresses"]=["https://www.gmail.com"] + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="EXCHANGE": + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Exchange" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ClientSecret", True, "UPDATEIT") + mytype=addParam(mytype,"ClientId", False, "UPDATEIT") + mytype=addParam(mytype,"ClientRefreshToken", True, "UPDATEIT") + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"emailAddress","*@*")) + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["emailAddress"] } + mytype["configuration"]["startingAddresses"]=[ "https://outlook.office365.com/exchange/"+config["cloudSourceConfiguration"]["emailAddress"] ] + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["emailAddress"], "password": "UPDATEIT" }} + mytype["password"]="TOCHANGE" + #mytype["sourceVisibility"] = "SHARED" + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="DROPBOX": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + email = getCloudSetting(config,"name","").replace('Dropbox - ','') + if email: + mytype["emailAddress"]=email + mytype["additionalInfos"]={"emailAddress": email } + mytype["accessToken"]="UPDATEIT" + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="TWITTER": + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="LITHIUM": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "serverAddress" in config["cloudSourceConfiguration"]: + mytype["serverAddress"]=config["cloudSourceConfiguration"]["serverAddress"] + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="SHAREPOINT": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "url" in config["cloudSourceConfiguration"]: + mytype["urls"]=[config["cloudSourceConfiguration"]["url"]] + mytype["crawlScope"]="SiteCollection" + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="SHAREPOINT_ONLINE": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tRefresh Token" + actionlist += "\n\tAdd Tennant name" + mytype["RefreshToken"]="UPDATEIT" + mytype["tenantname"]="UPDATEIT" + mytype["crawlScope"]="SiteCollection" + if "url" in config["cloudSourceConfiguration"]: + mytype["urls"]=[config["cloudSourceConfiguration"]["url"]] + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="SHAREPOINT_LEGACY": + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="YAMMER": + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="WEBSCRAPER": + actionlist += "\n\tAdjust webscraper configuration" + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="GOOGLE_DRIVE_DOMAIN_WIDE": + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="RSS": + if "uri" in config["cloudSourceConfiguration"]: + mytype["urls"]=[config["cloudSourceConfiguration"]["uri"]] + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="WEB": + a=0 + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="CONFLUENCE": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="CONFLUENCE2": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Confluence2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"IndexComments", False, getCloudSetting(config,"indexComments",True)) + mypersonal = getCloudSetting(config,"indexOnlyPersonalSpaces",False) + myglobal = getCloudSetting(config, "indexOnlyGlobalSpaces", False) + #Current/Archived is not present in V1 + mycurrent = getCloudSetting(config,"indexCurrentSpaces",True) + myarchived = getCloudSetting(config, "indexArchivedSpaces", False) + if mypersonal and myglobal: + myglobal = False + mypersonal = True + mytype=addParam(mytype,"IndexOnlyPersonalSpaces", False, getCloudSetting(config,"indexOnlyPersonalSpaces",mypersonal)) + mytype=addParam(mytype,"IndexArchivedSpaces", False, getCloudSetting(config, "indexArchivedSpaces", myarchived)) + mytype=addParam(mytype,"IndexOnlyGlobalSpaces", False, getCloudSetting(config, "indexOnlyGlobalSpaces", myglobal)) + mytype=addParam(mytype,"IndexAttachments", False, getCloudSetting(config, "indexAttachments", True)) + mytype=addParam(mytype,"IndexCurrentSpaces", False, getCloudSetting(config, "indexCurrentSpaces", mycurrent)) + mytype=addParam(mytype,"FilterSpaceRegex", False, getCloudSetting(config,"spaceFilter","")) + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"username","*@*")) + + if "username" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["username"] + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["username"], "password": "UPDATEIT" }} + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["username"] } + if "urls" in config["cloudSourceConfiguration"]: + mytype["configuration"]["startingAddresses"]= config["cloudSourceConfiguration"]["urls"] + mytype["password"]="TOCHANGE" + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="CONFLUENCE2_CLOUD": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Confluence2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"IndexComments", False, getCloudSetting(config,"indexComments",True)) + mypersonal = getCloudSetting(config,"indexOnlyPersonalSpaces",False) + myglobal = getCloudSetting(config, "indexOnlyGlobalSpaces", False) + #Current/Archived is not present in V1 + mycurrent = getCloudSetting(config,"indexCurrentSpaces",True) + myarchived = getCloudSetting(config, "indexArchivedSpaces", False) + if mypersonal and myglobal: + myglobal = False + mypersonal = True + mytype=addParam(mytype,"IndexOnlyPersonalSpaces", False, getCloudSetting(config,"indexOnlyPersonalSpaces",mypersonal)) + mytype=addParam(mytype,"IndexArchivedSpaces", False, getCloudSetting(config, "indexArchivedSpaces", myarchived)) + mytype=addParam(mytype,"IndexOnlyGlobalSpaces", False, getCloudSetting(config, "indexOnlyGlobalSpaces", myglobal)) + mytype=addParam(mytype,"IndexAttachments", False, getCloudSetting(config, "indexAttachments", True)) + mytype=addParam(mytype,"IndexCurrentSpaces", False, getCloudSetting(config, "indexCurrentSpaces", mycurrent)) + mytype=addParam(mytype,"FilterSpaceRegex", False, getCloudSetting(config,"spaceFilter","")) + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"username","*@*")) + if "username" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["username"] + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["username"], "password": "UPDATEIT" }} + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["username"] } + if "urls" in config["cloudSourceConfiguration"]: + mytype["configuration"]["startingAddresses"]= config["cloudSourceConfiguration"]["urls"] + mytype["password"]="TOCHANGE" + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="ORACLE_KNOWLEDGE": + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="YOUTUBE": + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="SITEMAP": + actionlist += "\n\tAdjust webscraper configuration" + if (getCloudSetting(config,"urlReplacementPattern","")): + actionlist += "\n\tUrl replacement is obsolete, add your extension script to solve it." + if (getCloudSetting(config,"oAuthProviderType","")): + actionlist += "\n\toAuthProviderType is obsolete, set form authentication in the source." + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="JIRA": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Jira2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ShallIndexWorkLogs", False, getCloudSetting(config,"indexWorkLogs",True)) + mytype=addParam(mytype,"ShallIndexComments", False, getCloudSetting(config,"indexComments",False)) + mytype=addParam(mytype,"ShallIndexAttachments", False, getCloudSetting(config, "indexAttachments", False)) + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"username","*@*")) + if "username" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["username"] + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["username"], "password": "UPDATEIT" }} + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["username"] } + if "startingAddress" in config["cloudSourceConfiguration"]: + mytype["configuration"]["startingAddresses"]= [ config["cloudSourceConfiguration"]["startingAddress"] ] + mytype["password"]="TOCHANGE" + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="JIRA_CLOUD": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Jira2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ShallIndexWorkLogs", False, getCloudSetting(config,"indexWorkLogs",True)) + mytype=addParam(mytype,"ShallIndexComments", False, getCloudSetting(config,"indexComments",False)) + mytype=addParam(mytype,"ShallIndexAttachments", False, getCloudSetting(config, "indexAttachments", False)) + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"username","*@*")) + if "username" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["username"] + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["username"], "password": "UPDATEIT" }} + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["username"] } + if "startingAddress" in config["cloudSourceConfiguration"]: + mytype["configuration"]["startingAddresses"]= [ config["cloudSourceConfiguration"]["startingAddress"] ] + mytype["password"]="TOCHANGE" + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="JIVE": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Jive" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"OnlyIndexPublishedContent", False, getCloudSetting(config,"indexOnlyPublishedItems",False)) + mytype=addParam(mytype,"ShallIndexGroups", False, getCloudSetting(config,"indexSocialGroups",True)) + mytype=addParam(mytype,"ShallIndexSystemBlogs", False, getCloudSetting(config, "indexSystemBlogs", True)) + mytype=addParam(mytype,"ShallIndexSpaces", False, getCloudSetting(config,"indexCommunities",True)) + mytype=addParam(mytype,"ShallIndexProjects", False, getCloudSetting(config,"indexProjects",True)) + mytype=addParam(mytype,"ShallIndexPeople", False, getCloudSetting(config, "indexUsers", False)) + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + mytype["additionalInfos"] = { "JiveInstanceAllowsAnonymousAccess": getCloudSetting(config, "allowsAnonymousAccess", True) } + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"username","*@*")) + if "username" in config["cloudSourceConfiguration"]: + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["username"], "password": "UPDATEIT" }} + mytype["emailAddress"]=config["cloudSourceConfiguration"]["username"] + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["username"] } + if "url" in config["cloudSourceConfiguration"]: + mytype["configuration"]["startingAddresses"]= [ config["cloudSourceConfiguration"]["url"] ] + mytype["password"]="TOCHANGE" + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="JIVE_CLOUD": + if "publicVisibility" in config["cloudSourceConfiguration"]: + if not config["cloudSourceConfiguration"]["publicVisibility"]: + actionlist += "\n\tWARNING: V1 has public Visibility set to false, check security in V2!!!" + actionlist += "\n\tAdd password" + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["email"]=config["cloudSourceConfiguration"]["emailAddress"] + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Jive" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"OnlyIndexPublishedContent", False, getCloudSetting(config,"indexOnlyPublishedItems",False)) + mytype=addParam(mytype,"ShallIndexGroups", False, getCloudSetting(config,"indexSocialGroups",True)) + mytype=addParam(mytype,"ShallIndexSystemBlogs", False, getCloudSetting(config, "indexSystemBlogs", True)) + mytype=addParam(mytype,"ShallIndexSpaces", False, getCloudSetting(config,"indexCommunities",True)) + mytype=addParam(mytype,"ShallIndexProjects", False, getCloudSetting(config,"indexProjects",True)) + mytype=addParam(mytype,"ShallIndexPeople", False, getCloudSetting(config, "indexUsers", False)) + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + mytype["additionalInfos"] = { "JiveInstanceAllowsAnonymousAccess": getCloudSetting(config, "allowsAnonymousAccess", True) } + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Specified" + mytype=addSecurityUser(mytype,getCloudSetting(config,"username","*@*")) + if "username" in config["cloudSourceConfiguration"]: + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": config["cloudSourceConfiguration"]["username"], "password": "UPDATEIT" }} + mytype["emailAddress"]=config["cloudSourceConfiguration"]["username"] + mytype["additionalInfos"]={"emailAddress": config["cloudSourceConfiguration"]["username"] } + if "url" in config["cloudSourceConfiguration"]: + mytype["configuration"]["startingAddresses"]= [ config["cloudSourceConfiguration"]["url"] ] + mytype["password"]="TOCHANGE" + + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="SALESFORCE": + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + actionlist += "\n\tImport field configuration" + #Must be done using RAW + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Salesforce2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + #remove objectsSchema + mytype["objectsSchema"]=[] + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ClientSecret", True, "UPDATEIT") + mytype=addParam(mytype,"ClientId", False, "UPDATEIT") + mytype=addParam(mytype,"ClientRefreshToken", True, "UPDATEIT") + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + mytype=addParam(mytype,"SchemaVersion", False, "LEGACY") + mytype=addParam(mytype,"IsSandbox", False, getCloudSetting(config,"sandbox", False)) + mytype=addParam(mytype,"UseRefreshToken", False, getCloudSetting(config,"useRefreshToken", True)) + mytype=addParam(mytype,"OAuthRefreshToken", True, "UPDATEIT") + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Merge" + #Weird token is in password of a useridentity + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": "unused", "password": "UPDATEIT" }} + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["emailAddress"] + if "username" in config["cloudSourceConfiguration"]: + mytype["additionalInfos"]={"salesforceOrg": "UPDATEIT", "schemaVersion": "LEGACY", "salesforceOrgName": "UPDATEIT", "salesforceUser":config["cloudSourceConfiguration"]["username"] } + mytype["configuration"]["startingAddresses"]=["http://www.salesforce.com"] + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="KNOWLEDGEBASE": + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + actionlist += "\n\tImport field configuration" + #Must be done using RAW + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Salesforce2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + #remove objectsSchema + mytype["objectsSchema"]=[] + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ClientSecret", True, "UPDATEIT") + mytype=addParam(mytype,"ClientId", False, "UPDATEIT") + mytype=addParam(mytype,"ClientRefreshToken", True, "UPDATEIT") + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + mytype=addParam(mytype,"SchemaVersion", False, "LEGACY") + mytype=addParam(mytype,"IsSandbox", False, getCloudSetting(config,"sandbox", False)) + mytype=addParam(mytype,"UseRefreshToken", False, getCloudSetting(config,"useRefreshToken", True)) + mytype=addParam(mytype,"OAuthRefreshToken", True, "UPDATEIT") + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Merge" + #Weird token is in password of a useridentity + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": "unused", "password": "UPDATEIT" }} + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["emailAddress"] + if "username" in config["cloudSourceConfiguration"]: + mytype["additionalInfos"]={"salesforceOrg": "UPDATEIT", "schemaVersion": "LEGACY", "salesforceOrgName": "UPDATEIT", "salesforceUser":config["cloudSourceConfiguration"]["username"] } + mytype["configuration"]["startingAddresses"]=["http://www.salesforce.com"] + a=0 + #----------------------------------------------------------------------------------------------------------------- + if config["type"]=="SALESFORCE_CONTENT": + actionlist += "\n\tAdd password" + actionlist += "\n\tRefresh Token" + actionlist += "\n\tImport field configuration" + #Must be done using RAW + mytype["MethodToUse"]="RAW" + mytype["crawlerInstancetype"] = "Connector.Salesforce2" + mytype["configuration"]={} + mytype = addAddressPattern(mytype) + #remove objectsSchema + mytype["objectsSchema"]=[] + + mytype["configuration"]["parameters"]={} + mytype=addParam(mytype,"PauseOnError", False, "true") + mytype=addParam(mytype,"ClientSecret", True, "UPDATEIT") + mytype=addParam(mytype,"ClientId", False, "UPDATEIT") + mytype=addParam(mytype,"ClientRefreshToken", True, "UPDATEIT") + mytype=addParam(mytype,"OrganizationId", False, v1_org_id) + mytype=addParam(mytype,"SchemaVersion", False, "LEGACY") + mytype=addParam(mytype,"IsSandbox", False, getCloudSetting(config,"sandbox", False)) + mytype=addParam(mytype,"UseRefreshToken", False, getCloudSetting(config,"useRefreshToken", True)) + mytype=addParam(mytype,"OAuthRefreshToken", True, "UPDATEIT") + if config["cloudSourceConfiguration"]["publicVisibility"]: + mytype=addSecurityEverybody(mytype) + mytype["configuration"]["sourceSecurityOption"] = "Specified" + else: + mytype["configuration"]["sourceSecurityOption"] = "Merge" + #Weird token is in password of a useridentity + mytype["configuration"]["userIdentities"]= { "UserIdentity": { "name": "token", "userName": "unused", "password": "UPDATEIT" }} + if "emailAddress" in config["cloudSourceConfiguration"]: + mytype["emailAddress"]=config["cloudSourceConfiguration"]["emailAddress"] + if "username" in config["cloudSourceConfiguration"]: + mytype["additionalInfos"]={"salesforceOrg": "UPDATEIT", "schemaVersion": "LEGACY", "salesforceOrgName": "UPDATEIT", "salesforceUser":config["cloudSourceConfiguration"]["username"] } + mytype["configuration"]["startingAddresses"]=["http://www.salesforce.com"] + a=0 + return mytype + +def translateVisibility(mytype): + fromRepo = { "PUBLIC":"SHARED","PRIVATE":"SECURED"} + for repo in fromRepo: + if repo == mytype: + return fromRepo[repo] + return mytype + +def transformV1ToV2(myconfig): + #Transform V1 source to V2 RAW format + #Check for problems in SFDC configs + restrictedfields = { "name","notifyOnRebuildCompleted","objectsSchema","owner","publicVisibility","rebuildRequired", "sourceType","sourceVisibility", "titleSelectionSequence","type","urls", "urlFilters"} + + report = {} + #print (myconfig["cloudSourceConfiguration"]) + report["id"]=myconfig["id"] + report["information"]={} + report["information"]["sourceId"] = myconfig["id"] + sourcename="" + if 'name' in myconfig["cloudSourceConfiguration"]: + sourcename = myconfig["cloudSourceConfiguration"]["name"] + else: + sourcename = myconfig["id"] + report["information"]["sourceName"] = sourcename + report["name"] = sourcename + if 'owner' in myconfig["cloudSourceConfiguration"]: + report["owner"] = myconfig["cloudSourceConfiguration"]["owner"] + report["sourceType"] = translatetype(myconfig["type"]) + report["sourceVisibility"] = translateVisibility(myconfig["visibility"]) + if 'urls' in myconfig["cloudSourceConfiguration"]: + report["urls"] = myconfig["cloudSourceConfiguration"]["urls"] + if 'urlFilters' in myconfig["cloudSourceConfiguration"]: + report["urlFilters"] = myconfig["cloudSourceConfiguration"]["urlFilters"] + + report["customParameters"] = {} + for field in myconfig["cloudSourceConfiguration"]: + if not field in restrictedfields: + report["customParameters"][field]=myconfig["cloudSourceConfiguration"][field] + report[field]=myconfig["cloudSourceConfiguration"][field] + report = fix(report,myconfig) + return report + + +def inspectSFDC(myconfig): + #Inspects SFDC custom settings + #Check for problems in SFDC configs + report = "" + if myconfig['cloudSourceConfiguration']: + if 'objectsSchema' in myconfig['cloudSourceConfiguration']: + for objects in myconfig['cloudSourceConfiguration']['objectsSchema']: + if objects['custom']==True: + body = "" + if 'body' in objects: + body = objects['body'] + report += "\n\tSFDC SCHEMA FOR OBJECT: "+objects['name']+", different body:\n\t"+body+"\n\t==================================================\n" + for fields in objects['fields']: + if fields['custom']==True: + report += "\n\tSFDC SCHEMA FOR OBJECT: "+objects['name']+"" + #facet, fieldName, freeText, label, multi, name, parentObjectName, mytype + report += "\n\tSFDC SCHEMA FOR FIELD : "+fields["name"]+"\n"+addLine() + report += "\tfieldName : "+fields["fieldName"]+"\n"+addLine() + report += "\tlabel : "+fields["label"]+"\n"+addLine() + report += "\ttype : "+fields["type"]+"\n"+addLine() + report += "\tparentObjectName : "+fields["parentObjectName"]+"\n"+addLine() + report += "\tfacet : "+str(fields["facet"])+"\n"+addLine() + report += "\tfreeText : "+str(fields["freeText"])+"\n"+addLine() + report += "\tmulti facet : "+str(fields["multi"])+"\n" + report += "\n\t==================================================\n" + + return report + + +def removeComments(string): + string = re.sub(re.compile("/\*.*?\*/",re.RegexFlag.MULTILINE|re.RegexFlag.DOTALL ) ,"" ,string) # remove all occurance streamed comments (/*COMMENT */) from string + string = re.sub(re.compile("//.*?\n" ,re.RegexFlag.MULTILINE|re.RegexFlag.DOTALL ) ,"" ,string) # remove all occurance singleline comments (//COMMENT\n ) from string + return string + + +def changeComments(string): + string = re.sub(re.compile("//(.*)[^;]\n" ) ,"// \\1 \n" ,string) # remove all occurance singleline comments (//COMMENT\n ) from string + string = re.sub(re.compile("/\*([\S\s]*?)\*/",re.RegexFlag.MULTILINE|re.RegexFlag.DOTALL ) ,"/* \\1 */" ,string) # remove all occurance streamed comments (/*COMMENT */) from string + #string = re.sub(re.compile("//(.*)\n" ) ,"// \\1\n" ,string) # remove all occurance singleline comments (//COMMENT\n ) from string + return string + +def toPython(string, filename): + global functionstocall + global finalreport + string = changeComments(string) + #print (string) + string = jsbeautifier.beautify(string) + #replace functions + addWriteOutput=False + addWriteHTMLOutput=False + #function (.*)[^\r]{ + #string = re.sub(re.compile("function (.*)[^\r]{",re.DOTALL ) ,"function \\1{" ,string) + """ string = re.sub(re.compile("function (.*)[^\r]{",re.DOTALL ) ,"def \\1:" ,string) + #var fields + #var fieldList = { + string = re.sub(re.compile("var (.*)=[ ]*{([\S\s][^}]*)}[;]*",re.DOTALL ) ,"\\1={\\2}" ,string) + #var + string = re.sub(re.compile("var ",re.DOTALL ) ,"" ,string) + # && + string = re.sub(re.compile(" && ",re.DOTALL ) ," and " ,string) + # { + string = re.sub(re.compile("{",re.DOTALL ) ,":" ,string) + # } + string = re.sub(re.compile("}",re.DOTALL ) ,"" ,string) """ + #fix regex + #var (\w)*[ ]*=[ ]*\/(.*); + string = re.sub(re.compile("var (\w*)[ ]*=[ ]*\/(.*)/(.*);" ) ,"\\1=r\"\\2\";" ,string) + #var + string = re.sub(re.compile("var (\w*);" ) ,"\\1=None;" ,string) + string = re.sub(re.compile("var " ) ,"" ,string) + + # + string = re.sub(re.compile("{}" ) ,"{\n//Empty\npass\n}" ,string) + + + + #PostConversion.Trace + string = re.sub(re.compile("PostConversion\.Trace\((.*),.?0\)" ) ,"log(\\1, 'Normal')" ,string) + string = re.sub(re.compile("PostConversion\.Trace\((.*),.?1\)" ) ,"log(\\1, 'Debug')" ,string) + string = re.sub(re.compile("PostConversion\.Trace\((.*),.?2\)" ) ,"log(\\1, 'Error')" ,string) + string = re.sub(re.compile("PostConversion\.Trace\((.*),.?(\w*)\)" ) ,"log(\\1, \\2)" ,string) + string = re.sub(re.compile("PostConversion\.Trace\((.*)\)" ) ,"log(\\1, 'Normal')" ,string) + string = re.sub(re.compile("PostConversion\.Trace" ) ,"log" ,string) + + #DocumentInfo.URI + string = re.sub(re.compile("DocumentInfo\.URI",re.RegexFlag.MULTILINE|re.RegexFlag.DOTALL ) ,"document.uri" ,string) + + #\(function (\w*)\((.*){([\s\S\n]*)\)\(\); + #print (string) + while re.search("\(function (\w*)\((.*){([\s\S\n]*)}[\n]*\)\(\);", string ): + string = re.sub(re.compile("\(function (\w*)\((.*){([\s\S\n]*)}[\n]*\)\(\);" ) ,"function \\1(\\2{\\3} \\1();" ,string) + while re.search("\(function\(\) {([\s\S\n]*)}[\n]*\)\(\)", string ): + string = re.sub(re.compile("\(function\(\) {([\s\S\n]*)}[\n]*\)\(\)" ) ,"\\1" ,string) + + + #Gather all the functions called which we might need to support + #\.(\w*)\( + for match in re.finditer('\.(\w*)\(', string): + if not match.group(1) in functionstocall: + functionstocall.append(match.group(1)) + finalreport += "\nFunction call found in JS file: "+(match.group(1)+ "-->"+filename) + #PostConversion.HTMLOutputToOverride.WriteString + if "PostConversion.HTMLOutputToOverride.WriteString" in string: + addWriteHTMLOutput = True + if "PostConversion.TextToOverride.WriteString" in string: + addWriteOutput = True + string = re.sub(re.compile("PostConversion.HTMLOutputToOverride.WriteString",re.RegexFlag.MULTILINE|re.RegexFlag.DOTALL ) ,"WriteHTMLOutput" ,string) + string = re.sub(re.compile("PostConversion.TextToOverride.WriteString",re.RegexFlag.MULTILINE|re.RegexFlag.DOTALL ) ,"WriteOutput" ,string) + #DocumentInfo.GetFieldValue + string = re.sub(re.compile('DocumentInfo\.GetFieldValue\(("?\w*"?)\);' ) ,"document.get_meta_data_value(\\1)[0]" ,string) + string = re.sub(re.compile("DocumentInfo\.(\w*)[ ]*=(.*);" ) ,"document.add_meta_data({'\\1':\\2})" ,string) + #DocumentInfo.SetFieldValue --> document.add_meta_data({"mylat2":str(i['Lat'])}) + #;$ + #string = re.sub(re.compile(";$",re.DOTALL ) ,"" ,string) + #Switch + #switch \((\w+):([\S\s\n]*)case ['\"]*(\w+)['\"]: + #first replace all case, then when nothing left, replace the switch + #print (string) + #print ("START") + while re.search(r"switch \((\w+)\) {([\S\s\n]*)case ['\"]*(\w+)['\"]:([\S\s\n]*?)break;", string ): + #print ("IN HERE") + string = re.sub(re.compile(r"switch \((\w+)\) {([\S\s\n]*)case (['\"]*\w+['\"]*):([\S\s\n]*?)break;" ) ,"switch (\\1) {\\2if (\\1==\\3) {\n\\4\n}" ,string) + #replace the switch itself + string = re.sub(re.compile(r"switch \((\w+)\)") ,"a=0;\nif (a==0) " ,string) + + + string = jsbeautifier.beautify(string) + #print (string) + string = jiphy.to.python(string) + string = re.sub(re.compile("\);") ,")" ,string) + string = re.sub(re.compile("\(\:") ,":" ,string) + + #print (string) + string = re.sub(re.compile("DocumentInfo\.SetFieldValue\((.*),(.*)\)") ,"document.add_meta_data({\\1:\\2})" ,string) + #fix for loops + #for (\w*) = (\w*); (\w+) (\S+) ([\w.]+); ([\w+]+): + string = re.sub(re.compile("for (\w*) = (\w*); (\w+) (\S+) ([\w.]+); ([\w+]+):") ,"for \\1 in range(\\2,\\5): #for \\1 = \\2;\\3 \\4 \\5; \\6" ,string) + string = re.sub(re.compile("for (\w*) = ([\w\.\(\)]*) (\w*) (\W*) ([\w\.\(\)]*) (\S*):") ,"for \\1 in range(\\2, \\5): #for \\1 = \\2;\\3 \\4 \\5; \\6" ,string) + #fix length + #(\w*).length + string = re.sub(re.compile("(\w*).length") ,"len(\\1)" ,string) + #fix lower/uppercase + string = re.sub(re.compile("toLowerCase\(\)") ,"lower()" ,string) + string = re.sub(re.compile("toUpperCase\(\)") ,"upper()" ,string) + #fix replace + #\.replace\(\/(.*)\/(.*), + string = re.sub(re.compile("(\w*)\.(.*)\.replace\(\/(.*)\/(.*), (.*)\)") ,"\\1.\\2\r\\1=re.sub('\\3',\\5,\\1) #\\4\n\n" ,string) + string = re.sub(re.compile("(this)\.replace\(\/(.*)\/(.*), (.*)\)") ,"re.sub('\\2',\\4,\\1) #\\3 FIX the this reference\n" ,string) + string = re.sub(re.compile("(\w*)\.replace\(\/(.*)\/(.*), (.*)\)") ,"re.sub('\\2',\\4,\\1) #\\3\n" ,string) + string = re.sub(re.compile("(\w*)\.replace\((.*), (.*)\)"), "re.sub(\\2,\\3,\\1) #\\3\n" ,string) + #ClickableURI + #DocumentInfo\.ClickableURI + string = re.sub(re.compile("DocumentInfo\.ClickableURI") ,"document.get_meta_data_value('clickableuri')[0]" ,string) + string = re.sub(re.compile("DocumentInfo\.Title") ,"document.get_meta_data_value('title')[0]" ,string) + string = re.sub(re.compile("DocumentInfo\.FileType") ,"document.get_meta_data_value('filetype')[0]" ,string) + string = re.sub(re.compile("DocumentInfo\.Date") ,"document.get_meta_data_value('date')[0]" ,string) + + string = re.sub(re.compile("PostConversion\.HTMLOutput\.SeekReadPointerInBytes\(\d+\)"), "", string) + string = re.sub(re.compile("PostConversion\.HTMLOutput\.ReadByteString\(.*"),'document.get_data_stream("body_html", "converter").read().replace("\\\\n", "")',string) + + string = re.sub(re.compile("PostConversion\.Text\.ReadString\(.*"),'document.get_data_stream("body_text", "converter").read().replace("\\\\n", "")',string) + #search + #\.search\( + string = re.sub(re.compile("\.search\(") ,".index(" ,string) + #print (string) + #concat + string = re.sub(re.compile("(\w*) = (\w*)\.concat\((\S*)\)") ,"\\1 += \\3" ,string) + #shift + string = re.sub(re.compile("\.shift\(\)") ,".pop(0)" ,string) + #Round + string = re.sub(re.compile("Math\.round\((.[^\)]*)\)") ,"round(\\1)" ,string) + #fix if not + string = re.sub(re.compile("([\w\.]*) not = (\w*)") ,"not \\1 == \\2" ,string) + #indexof + string = re.sub(re.compile(".indexOf\((\w+)\)") ,".find(\\1)" ,string) + string = re.sub(re.compile(".indexOf\((\w+), (\w+)\)") ,".find(\\1,\\2)" ,string) + string = re.sub(re.compile("\.indexOf\("),".find(",string) + #regexp + string = re.sub(re.compile("new RegExp\((.*[\"]), \"(\w+)\"\)") ,"r\\1 #\\2" ,string) + string = re.sub(re.compile("new RegExp\((.*), \"(\w+)\"\)") ,"re.compile(\\1) #\\2" ,string) + string = re.sub(re.compile("new RegExp\((.*[\"])\)") ,"r\\1" ,string) + string = re.sub(re.compile("new RegExp\((.*)\)") ,"re.compile(\\1)" ,string) + #match + string = re.sub(re.compile("(\w+).match\(\/(.*)\/(.*)\):"),"re.search(\"\\2\",\\1):", string) + string = re.sub(re.compile("(\w+).match\((.*)\)"),"re.search(\\1,\\2)", string) + #exec + string = re.sub(re.compile("(\w+).exec\((.*)\)"),"re.search(\\1,\\2)", string) + #join + string = re.sub(re.compile("(\w+)\.join\((.*?)\)"),"\\2.join(\\1)", string) + #lastindexof + string = re.sub(re.compile("(\w+)\.lastIndexOf\((.*)\)"),"\\1.rindex(\\2)", string) + #substring + string = re.sub(re.compile("(\w+)\.substring\((.*),(.*)\)"),"\\1[\\2:\\3]", string) + string = re.sub(re.compile("(\w+)\.substring\((.*)\)"),"\\1[\\2:]", string) + #toString + string = re.sub(re.compile("(\w*)\.toString\(\)"),"str(\\1)", string) + #typeof + string = re.sub(re.compile("typeof (.*) (is|==) ([not ]*)'(\w+)'"),"\\3 instance(\\1,\\4)", string) + #long text + string = re.sub(re.compile("' \+\n( *)'"),"\\\\n\"\n\\1\"", string) + string = re.sub(re.compile("'(.*\")\n([\S\s\n]*?)'"),"(\"\\1\n\\2\")", string) + #string = re.sub(re.compile("\"(.*?)\"(.*?)\"(.*)\n"),"\"\\1\\\"\\2\\\"\\3\n", string) + string = re.sub(re.compile("=\"(.*?)\""),"=\\\"\\1\\\"", string) + string = re.sub(re.compile("= r \""),"= r\"", string) + string = re.sub(re.compile("RegExp.\$(\d)"),"CHANGETHIS.group(\\1)", string) + #or + string = re.sub(re.compile("\|\|[ ]*\n" ) ," or " ,string) + string = re.sub(re.compile("//(.*)[^;]\n" ) ,"# \\1 \n" ,string) # remove all occurance singleline comments (//COMMENT\n ) from string + string = re.sub(re.compile(" is not None:"), ":", string) + addcode = "import re\n\n" + #add scripts + if addWriteOutput: + addcode += ("def WriteOutput(text):\n# Get the Body Text holder\n" + " bodytext = document.DataStream('body_text')\n" + " # Write to the bodytext holder\n" + " bodytext.write(text)\n" + " # Add the datastream back in\n" + " document.add_data_stream(bodytext)\n") + if addWriteHTMLOutput: + addcode += ("def WriteHTMLOutput(text):\n# Get the Body HTML holder\n" + " bodytext = document.DataStream('body_html')\n" + " # Write to the bodytext holder\n" + " bodytext.write(text)\n" + " # Add the datastream back in\n" + " document.add_data_stream(bodytext)\n") + string = addcode + "\n\n"+string + + #print (string) + return string + +def checkScriptFields(myconfig,filename, sourcename): + #Checks existing fields, any custom scripts will be outputted + global actionlist + report = "" + #print (myconfig) + for field in myconfig: + values = myconfig[field] + #print (fieldcontent) + #for values in fieldcontent: + #print (values) + if values['contentType']=="CUSTOM_SCRIPT": + report += "\n\tCUSTOM SCRIPT: "+values['name']+", in: "+filename+"_"+values['name']+".js\n" + report += "\n\tCUSTOM SCRIPT: "+values['name']+", in: "+filename+"_"+values['name']+".py\n" + #report += values['customScriptContent'] + #report += "\n==================================================\n" + current = values['customScriptContent'] + mytype="" + #Check for big problems + if 'Mutex.AcquireLock' in current: + mytype = "AQLOCK_"+mytype + if 'GetGlobalVariable' in current: + mytype = "GLOBAL_"+mytype + if 'ActiveXObject' in current: + mytype = "ACTIVEX_"+mytype + file = open("output/"+filename+"_"+values['name']+".js","w" ) + file.write(values['customScriptContent']) + file.close() + file = open("output/"+mytype+filename+"_"+values['name']+".py","w" ) + file.writelines(toPython(current, mytype+filename+"_"+values['name']+".py")) + file.close() + actionlist += "\n\n\tMigrate/validate python script: "+mytype+filename+"_"+values['name']+".py" + actionlist += "\n\tAdd it as IPE to your Cloud V2 Organization" + actionlist += "\n\tAssign it to source: "+sourcename+"\n" + #Now create the python translation + #Python translation + return report + +def yes_or_no(question): + reply = str(input(question+' (y/n): ')).lower().strip() + if reply[0] == 'y': + return True + if reply[0] == 'n': + return False + else: + return yes_or_no("Uhhhh... please enter ") + +def createSource(client, myconfig): + sourceid="" + response = client.source_create(myconfig) + if 'id' in response: + sourceid = response["id"] + else: + sourceid = "ERROR: "+response["message"] + return sourceid + +if __name__ == '__main__': + + import doctest + if doctest.testmod().failed > 0: + exit(-1) + finalreport="" + actionlist="" + if not os.path.exists("output"): + os.mkdir("output") + + parser = argparse.ArgumentParser(description='Migrate fields configuration from CloudV2 to CloudV2') + parser.add_argument('--env', required=True, type=Environment, choices=list(Environment)) + parser.add_argument('--v1_org_id', required=True) + #parser.add_argument('--v1_source_name', required=False) + parser.add_argument('--v1_access_token', required=True) + parser.add_argument('--v2_org_id', required=True) + #parser.add_argument('--v2_source_id', required=True) + parser.add_argument('--v2_access_token', required=True) + parser.add_argument('--delete_fields', action='store_true') + opts = parser.parse_args() + functionstocall = [] + # args + env = opts.env + v1_org_id = opts.v1_org_id + #v1_source_name = opts.v1_source_name + v1_access_token = opts.v1_access_token + v2_org_id = opts.v2_org_id + #v2_source_id = opts.v2_source_id + v2_access_token = opts.v2_access_token + delete_fields = opts.delete_fields + + v1_client = CloudV2(env, v1_org_id, v1_access_token) + v2_client = CloudV2(env, v2_org_id, v2_access_token) + + v1_sources = v1_client.sources_get() + if "message" in v1_sources: + print ("Bad stuff is happening in V1 (old): "+json.dumps(v1_sources)) + exit + v2_sources = v2_client.sources_get() + if "message" in v2_sources: + print ("Bad stuff is happening in V2 (new): "+json.dumps(v2_sources)) + exit + + + + #remove sources from v2 + print("Removing V2 (new org) sources.") + if yes_or_no("Delete V2 (new org) sources first?"): + for source in v2_sources: + result = json.dumps(v2_client.sources_delete(source["id"])) + if 'sourceType' in result: + print("Removing: "+source["id"]+", DONE") + else: + print("Removing: "+source["id"]+", ERROR: "+result) + input("Press Enter to continue... (wait a bit so all sources in v2 are removed") + + finalreport+="Migration of:\n" + finalreport+="Organization: "+v1_org_id+" (V1, old org) --> "+v2_org_id+" (V2, new org)\n" + actionlist = finalreport + + + #Report sources found + for source in v1_sources['sources']: + sourcename="" + if 'name' in source: + sourcename = source['name'] + else: + sourcename = source["id"] + #input("Press Enter to continue... Creating source: "+sourcename) + actionlist += "\n\nSource: "+sourcename + finalreport+="\n\n====================================================================================\nSource name : "+sourcename+"\n" + print (f'\n=================================================\nSource name: {sourcename}') + print (f'Source id: {source["id"]}') + print (f'V1 config:') + raw = v1_client.source_get(source["id"]) + v1_fields = v1_get_fields_by_name(v1_client.fields_get_for_source(source["id"])) + #print(f'Fields present in CloudV1 ({len(v1_fields)}): {v1_fields}') + adddisable="_DISABLED_" + disabled = False + if raw["platformStatus"] =="DISABLED": + print (f'Skipping, disabled') + finalreport+= "\tSOURCE DISABLED, SKIPPING" + actionlist += "\n\tDisabled, not migrated" + filename = re.sub('[^A-Za-z0-9_]+', ' ',v1_org_id+"_"+sourcename+adddisable) + disabled = True + else: + filename = re.sub('[^A-Za-z0-9_]+', ' ',v1_org_id+"_"+sourcename) + file = open("output/"+filename+".json","w" ) + file.write(json.dumps(raw, indent=4, sort_keys=True)) + file.close() + if not disabled: + #print (f'=========================================================================================') + #print (f'V2 config:') + #v2source = transformV1ToV2(raw) + v2source = raw + report="" + addfile="" + addSource=True + if v2source["sourceType"]=="BAD": + addfile +="_NO_LONGER_SUPPORTED_" + report += "\tSource: "+sourcename+" NO LONGER SUPPORTED!!!\n\n" + actionlist += "\n\tNO LONGER SUPPORTED, contact PS" + addSource=False + #finalreport+=" Remark : "+addfile+"\n" + if v2source["sourceType"]=="MAN": + addfile+="_CANNOT BE CREATED DO IT MANUAL_" + report += "\tSource: "+sourcename+" CANNOT BE CREATED, DO IT MANUAL!!!\n\n" + actionlist += "\n\tCannot be automatically converted, must be manually created" + addSource=False + #finalreport+=" Remark : "+addfile+"\n" + if raw["editable"]==False: + addfile+="_BACKEND CHANGES CONTACT OPS_" + report += "\tSource: "+sourcename+" HAS BACKEND CHANGES, CONTACT OPS!!!\n\n" + actionlist += "\n\tHas backend changes, must be manually configured, contact OPS for changes." + addSource=False + #finalreport+=" Remark : "+addfile+"\n" + file = open("output/"+filename+addfile+"_V2.json","w" ) + file.write(json.dumps(v2source, indent=4, sort_keys=True)) + file.close() + + #SFDCChanges = inspectSFDC(raw) + #if SFDCChanges: + # finalreport+= "\tSalesforce source contains custom settings, see REMARKS file.\n" + # actionlist += "\n\t(Salesforce) contains custom settings, see REMARKS file, adjust your configuration." + + #report += checkScriptFields(v1_fields,filename, sourcename) + finalreport+= report + if report: + file = open("output/"+filename+"_REMARKS.txt","wb" ) + file.write(report.encode('utf-8')) + file.close() + #Create source in v2 + #Get source id + if addSource: + v2_source_id = createSource(v2_client, v2source) + if "ERROR" in v2_source_id: + finalreport+= "\tERROR AT CREATING SOURCE IN V2: "+v2_source_id + print (f'Error: {v2_source_id}') + actionlist += "\n\tCould not be created, create it manually." + else: + finalreport+= "\tNew source id in V2: "+v2_source_id + print (f'New ID: {v2_source_id}') + #Transfer fields from v1 to v2 + print("Adding fields to V2") + v2_mappings = v2_get_mappings_fieldname(v2_client.mappings_get(v2_source_id)) + common_fields = [v2_mapping for v2_mapping in v2_mappings if v2_mapping in v1_fields.keys()] + #print(f'Common field names between v1 and v2 ({len(common_fields)}): {common_fields}') + v2_fields_by_name = v2_get_fields_in_use(v2_client.fields_get(), v2_mappings) + #print(f'Fields to compare in v2 ({len(v2_fields_by_name)}): {v2_fields_by_name}') + fields_difference_to_apply = get_fields_differences(v1_fields, v2_fields_by_name) + #print(f'Found {len(fields_difference_to_apply)} fields to update: {fields_difference_to_apply}') + + v2_fields_updated = v2_get_updated_fields(fields_difference_to_apply) + if v2_fields_updated: + #print(f'CloudV2 fields to update ({len(v2_fields_updated)}): {v2_fields_updated}') + finalreport += "\n\tFields updated: "+json.dumps(v2_fields_updated) + result = v2_client.fields_update(v2_fields_updated) + if result==None: + finalreport += "\n\tFields succesfully added" + else: + finalreport += "\n\tError: " + json.dumps(result) + actionlist += "\n\tField creation failed, check logs." + else: + print('No fields to update.') + #------------------------------------------------------------------------ + #When Salesforce, call Evict to force the reload of the objectschema + #/sources/src/configuration/salesforce/evict + if "crawlerInstancetype" in v2source: + if v2source["crawlerInstancetype"]=="Connector.Salesforce2": + finalreport += "\n\tSalesforce Source must be Evicted, see actionlist." + #result = v2_client.source_evict(v2_source_id) + actionlist += f"\n\tAfter Refreshing token, execute in Swagger: https://platform.cloud.coveo.com/rest/organizations/{v2_org_id}/sources/{v2_source_id}/configuration/salesforce/evict." + #------------------------------------------------------------------------ + #Get Schedules and create them + #First get the just created schedule and remove it + v2_schedules = v2_client.schedule_get(v2_source_id) + for schedule in v2_schedules: + print ("Removing schedule: "+schedule['id']) + v2_client.schedule_delete(v2_source_id, schedule['id']) + #Removed, get the v1 schedules + v1_schedules = v1_client.schedule_get(source["id"]) + for schedule in v1_schedules: + #schedule["scheduleType"]="SOURCE" + #schedule["refreshType"]=schedule["type"] + #schedule["type"]="" + #set enabled to false + schedule["enabled"]=False + schedule["id"]="" + #print (json.dumps(schedule)) + addedschedule= v2_client.schedule_create(v2_source_id, schedule) + if 'id' in addedschedule: + finalreport += "\n\tNew schedule added: "+addedschedule['id'] + actionlist += "\n\tEnable schedule with id: "+addedschedule['id'] + else: + finalreport += "\n\tError in adding schedule: "+schedule['name'] + actionlist += "\n\tSchedule creation failed, check logs." + + #Create custom fields in v2 + #print (f'=========================================================================================') + + #Add custom fields + finalreport+="\n====================================================================================\n" + dry_run=False + if yes_or_no("Continue with fields and pipelines?"): + v1_user_fields = [field for field in v1_client.fields_get() if v1_field_is_user(field)] + v1_user_fields_unique = v1_get_unique_fields(v1_user_fields) + print (v1_user_fields_unique) + copy_user_fields(v1_user_fields_unique, v2_client, dry_run) + print('All users fields copied.') + + v1_fields_mapping = list(itertools.chain.from_iterable([field_list[1]['mappings'] for field_list in v1_user_fields_unique])) + v2_create_mapping_from_v1_fields(v2_client, v1_client.sources_get(), v1_fields_mapping, v2_client.sources_get(), dry_run) + finalreport+= "\n\nUser Fields updated: "+json.dumps(v1_fields_mapping).replace('{"name":','\n{"name":') + print('All mappings created.') + + + + #Pipelines ----------------------------------------------------------------------------------------------- + #{'id': 'b1bf1ba5-0d6d-4c0b-923c-df388e4b0f87', 'name': 'Copy of besttechCommunityTuned (1)', 'isDefault': False, + # 'description': 'Pipeline for the besttech community with Reveal', 'filter': None, 'splitTestName': None, + # 'splitTestTarget': None, 'splitTestRatio': None, 'splitTestEnabled': False, 'condition': None, 'position': 27, + # 'last_modified_by': None, 'created_by': None} + + actionlist += "\n=====================================================================" + actionlist += "\nQuery Pipelines\n" + print('Pipelines.') + print ('Deleting V2 pipelines') + defaultpipelineid="" + v2_pipelines = v2_client.pipelines_get() + #if yes_or_no("Delete V2 pipelines first?"): + for pipeline in v2_pipelines: + #print (pipeline) + if (pipeline["isDefault"]): + defaultpipelineid=pipeline["id"] + v2_client.pipeline_delete(pipeline["id"]) + + #default pipeline is normally already completely empty so we do not have to do anything + + print ("Create condition statements") + v2_statements = v2_client.statements_get() + for statement in v2_statements["statements"]: + #print (statement) + print ("Deleting condition statement "+statement["id"]) + v2_client.statement_delete(statement["id"]) + + v1_statements = v1_client.statements_get() + for statement in v1_statements["statements"]: + #print (statement) + print ("Creating condition statement "+statement["id"]) + response = v2_client.statement_create(statement) + if 'id' in response: + finalreport += "Created condition statement: "+statement["id"]+" in V2 with ID: "+response["id"] + else: + pipelineid = "ERROR: "+response["message"] + print (pipelineid) + finalreport += "\nERROR in creating condition statement: "+statement["id"]+", error: "+response["message"] + actionlist += "\nERROR in creating condition statement: "+statement["id"]+", CHECK LOGS" + + + v1_pipelines = v1_client.pipelines_get() + for pipeline in v1_pipelines: + #Create pipeline in V2 + #pipeline["id"]="" + print ("Creating pipeline "+pipeline["name"]) + #A/B Tests targets are not supported + #if pipeline["splitTestTarget"]: + # pipeline["splitTestTarget"] = None + # pipeline["splitTestName"] = None + # pipeline["splitTestRatio"] = None + # pipeline["splitTestEnabled"] = False + # actionlist += "\n\tWARNING: A/B tests are defined in pipeline: "+pipeline["name"]+", create them manually!!!" + # finalreport += "\n\tWARNING: A/B tests are defined in pipeline: "+pipeline["name"]+", create them manually!!!" + # #print (json.dumps(pipeline)) + pipelineid="" + goToFinalReport = False + if pipeline["name"]=="default": + pipelineid=defaultpipelineid + else: + response = v2_client.pipeline_create(pipeline) + if 'id' in response: + pipelineid = response["id"] + goToFinalReport =True + if goToFinalReport or pipeline["name"]=="default": + #pipelineid = response["id"] + finalreport += "\nCreated pipeline: "+pipeline["name"]+" in V2 with ID: "+pipelineid + #get all statements + v1_statements = v1_client.pipeline_statements_get(pipeline['id']) + for statement in v1_statements['statements']: + # {'id': 'c8d1b77f-a2b3-475d-9b85-ff5d144e47d9', 'description': '', 'feature': 'top', + # 'definition': 'when $query contains `enable netflix` then top `@sysurihash="YShRcUQrh1TvHQ3E"`', + # 'parent': {'id': '4c9a2e17-f04a-4fbb-a99c-a71813d05b80', 'description': '', 'definition': 'when $searchHub is "AgentPanel"', + # 'detailed': {'condition': {'operator': 'is', 'left': {'object': 'searchHub'}, 'right': 'AgentPanel'}}, 'childrenCount': 0, + # 'feature': 'when', 'parent': None, 'condition': None, 'position': 0, 'ready': True}, + # 'condition': {'id': '4c9a2e17-f04a-4fbb-a99c-a71813d05b80', 'description': '', 'definition': 'when $searchHub is "AgentPanel"', + # 'detailed': {'condition': {'operator': 'is', 'left': {'object': 'searchHub'}, 'right': 'AgentPanel'}}, + # 'childrenCount': 0, 'feature': 'when', 'parent': None, 'condition': None, 'position': 0, 'ready': True}, + # 'position': 14, 'ready': False, 'detailed': {'condition': {'operator': 'contains', 'left': {'object': 'query'}, 'right': 'enable netflix'}, + # 'statement': {'expressions': ['@sysurihash="YShRcUQrh1TvHQ3E"']}}, 'childrenCount': 0} + #Create statement in V2 + #print (statement) + #Get The details + statement = (v1_client.pipeline_statement_details_get(pipeline["id"], statement["id"])) + if "parent" in statement: + if statement["parent"]: + if "id" in statement["parent"]: + statement["parent"]=statement["parent"]["id"] + print ("Creating pipeline - statement "+pipeline["name"]+"("+pipelineid+"), "+statement["id"]) + #statement["id"]="" + #print (json.dumps(statement)) + response = v2_client.pipeline_statement_create(pipelineid, statement) + if 'id' in response: + statementid = response["id"] + finalreport += "\nCreated statement: "+pipeline["name"]+" in V2 with ID: "+statementid + else: + statementid = "ERROR: "+response["message"] + print (statementid) + finalreport += "\nERROR in creating query pipeline - STATEMENT: "+pipeline["name"]+" ("+pipeline["id"]+"), error: "+response["message"] + actionlist += "\nERROR in creating query pipeline - STATEMENT: "+pipeline["name"]+" ("+pipeline["id"]+"), CHECK LOGS" + + + else: + pipelineid = "ERROR: "+response["message"] + print (pipelineid) + finalreport += "\nERROR in creating query pipeline: "+pipeline["name"]+", error: "+response["message"] + actionlist += "\nERROR in creating query pipeline: "+pipeline["name"]+", CHECK LOGS" + + + + finalreport+="\n====================================================================================\n" + finalreport+="\nDimensions\n" + print ('Migrate dimensions') + v1_dimensions = v1_client.dimensions_get() + for dimension in v1_dimensions: + #print (json.dumps(dimension)) + # { "type": "TEXT", + # "displayName": "Facet Value", + # "apiNames": [ "CUSTOM_EVENTS.C_FACETVALUE" ], + # "returnName": "custom_events.c_facetvalue", + # "custom": true, + # "availableInVisit": true, + # "eventTypes": [ "custom_events" ], + # "status": "AVAILABLE" }, + + #{ + # "type": "TEXT", + # "displayName": "string" + # } + if dimension["custom"]: + name = dimension["returnName"] + events = dimension["eventTypes"] + myevent = "" + for event in events: + #Create Dimension + myevent+="&event="+event + mydimension={} + mydimension["type"] = dimension["type"] + mydimension["displayName"] = dimension["displayName"] + if '.' in name: + name = name.split('.')[1] + #print (name) + + result = v2_client.dimension_create(name,myevent,mydimension) + if 'type' in result: + finalreport += '\nDimension '+dimension["displayName"]+', created' + else: + finalreport += '\nERROR in creating Dimension '+dimension["displayName"]+', '+result["message"] + print ('\nERROR in creating Dimension '+dimension["displayName"]+', '+result["message"]) + actionlist += '\nERROR in creating Dimension '+dimension["displayName"]+', CHECK LOGS' + + + + print('Migration completed.') + finalreport+="\n====================================================================================\n" + finalreport += "Actions needed:\n" + finalreport += actionlist + finalreport+="\n====================================================================================\n" + file = open("output/"+v1_org_id+"_FINALREPORT.txt","wb" ) + file.write(finalreport.encode('utf-8')) + file.close() + file = open("output/"+v1_org_id+"_ACTIONLIST.txt","wb" ) + file.write(actionlist.encode('utf-8')) + file.close() + print ("Ready...") + print("Functions to call:") + for function in functionstocall: + print (function) +""" + v1_source_id = v1_get_source_id(v1_sources['sources'], v1_source_name) + + v2_mappings = v2_get_mappings_fieldname(v2_client.mappings_get(v2_source_id)) + print(f'Mapping present in CloudV2 ({len(v2_mappings)}): {v2_mappings}') + + common_fields = [v2_mapping for v2_mapping in v2_mappings if v2_mapping in v1_fields.keys()] + print(f'Common field names between v1 and v2 ({len(common_fields)}): {common_fields}') + + v2_fields_by_name = v2_get_fields_in_use(v2_client.fields_get(), v2_mappings) + print(f'Fields to compare in v2 ({len(v2_fields_by_name)}): {v2_fields_by_name}') + + fields_difference_to_apply = get_fields_differences(v1_fields, v2_fields_by_name) + print(f'Found {len(fields_difference_to_apply)} fields to update: {fields_difference_to_apply}') + + v2_fields_updated = v2_get_updated_fields(fields_difference_to_apply) + if v2_fields_updated: + print(f'CloudV2 fields to update ({len(v2_fields_updated)}): {v2_fields_updated}') + v2_client.fields_update(v2_fields_updated) + else: + print('No fields to update.') + print('Migration completed.') + + #Deleting unused fields + if delete_fields: + unused_fields = get_unused_fields(v2_client.fields_get_with_mappings()) + print('Deleting fields ' + unused_fields) + v2_client.fields_delete(unused_fields) + print('Field deletion completed.')""" \ No newline at end of file