From 86c2763f143329a06b5df7c524b666283cd91003 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Tue, 29 Aug 2023 13:14:26 -0400 Subject: [PATCH 01/11] feat(salesforce): Add functions to get spam user report, get sustainer report and convert salesforce 15char id to 18char id [sc-20303] --- sefaria/helper/crm/salesforce.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sefaria/helper/crm/salesforce.py b/sefaria/helper/crm/salesforce.py index 8f850d803f..2d84f5f4ec 100644 --- a/sefaria/helper/crm/salesforce.py +++ b/sefaria/helper/crm/salesforce.py @@ -145,3 +145,43 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en", except: return False return res + + def get_spam_users(self): + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SPAM_REPORT}" + res = self.post(endpoint) + return res + + def get_sustainers(self): + pass + + def sf15to18(self, id): + # from https://github.com/mslabina/sf15to18/blob/master/sf15to18.py + if not id: + raise ValueError('No id given.') + if not isinstance(id, str): + raise TypeError('The given id isn\'t a string') + if len(id) == 18: + return id + if len(id) != 15: + raise ValueError('The given id isn\'t 15 characters long.') + + # Generate three last digits of the id + for i in range(0, 3): + f = 0 + + # For every 5-digit block of the given id + for j in range(0, 5): + # Assign the j-th chracter of the i-th 5-digit block to c + c = id[i * 5 + j] + + # Check if c is an uppercase letter + if c >= 'A' and c <= 'Z': + # Set a 1 at the character's position in the reversed segment + f += 1 << j + + # Add the calculated character for the current block to the id + id += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ012345'[f] + + return id + + From 088eef3d912f8a8775ce2d160a3e0a206181afcd Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Tue, 29 Aug 2023 13:14:45 -0400 Subject: [PATCH 02/11] feat(build): Add reports for spam users and sustainers to settings [sc-20303] --- .../templates/configmap/local-settings-file.yaml | 2 ++ sefaria/local_settings_example.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml index a3b00fbe4b..0098f0e8ff 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml @@ -183,6 +183,8 @@ data: SALESFORCE_BASE_URL = os.getenv("SALESFORCE_BASE_URL") SALESFORCE_CLIENT_ID = os.getenv("SALESFORCE_CLIENT_ID") SALESFORCE_CLIENT_SECRET = os.getenv("SALESFORCE_CLIENT_SECRET") + SALESFORCE_SPAM_REPORT = os.getenv("SALESFORCE_SPAM_REPORT") + SALESFORCE_SUSTAINERS_REPORT = os.getenv("SALESFORCE_SUSTAINERS_REPORT") DISABLE_INDEX_SAVE = False diff --git a/sefaria/local_settings_example.py b/sefaria/local_settings_example.py index f6b1fbf201..9f1d700800 100644 --- a/sefaria/local_settings_example.py +++ b/sefaria/local_settings_example.py @@ -191,6 +191,8 @@ SALESFORCE_BASE_URL = "" SALESFORCE_CLIENT_ID = "" SALESFORCE_CLIENT_SECRET = "" +SALESFORCE_SPAM_REPORT = "" +SALESFORCE_SUSTAINERS_REPORT = "" # Issue bans to Varnish on update. USE_VARNISH = False From a110e539599c0ff44fa5079dd5e7b493227e0ccd Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Wed, 30 Aug 2023 14:39:15 -0400 Subject: [PATCH 03/11] feat(salesforce-script): Add fxs to create and find sf jobs [sc-20303] --- sefaria/helper/crm/salesforce.py | 65 ++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/sefaria/helper/crm/salesforce.py b/sefaria/helper/crm/salesforce.py index 2d84f5f4ec..98ef5b6e41 100644 --- a/sefaria/helper/crm/salesforce.py +++ b/sefaria/helper/crm/salesforce.py @@ -6,17 +6,19 @@ from sefaria.helper.crm.crm_connection_manager import CrmConnectionManager from sefaria import settings as sls + class SalesforceConnectionManager(CrmConnectionManager): def __init__(self): CrmConnectionManager.__init__(self, sls.SALESFORCE_BASE_URL) self.version = "56.0" + self.bulk_api_version = "54.0" self.resource_prefix = f"services/data/v{self.version}/sobjects/" def create_endpoint(self, *args): return f"{sls.SALESFORCE_BASE_URL}/{self.resource_prefix}{'/'.join(args)}" def make_request(self, request, **kwargs): - for attempt in range(0,3): + for attempt in range(0, 3): try: return request(**kwargs).json() except Exception as e: @@ -25,7 +27,7 @@ def make_request(self, request, **kwargs): def get(self, endpoint): headers = {'Content-type': 'application/json', 'Accept': 'application/json'} - return self.session.get(endpoint, headers) + return self.session.get(endpoint, headers=headers) def post(self, endpoint, **kwargs): headers = {'Content-type': 'application/json', 'Accept': 'application/json'} @@ -37,7 +39,7 @@ def patch(self, endpoint, **kwargs): def _get_connection(self): access_token_url = "%s/services/oauth2/token?grant_type=client_credentials" % self.base_url - base64_auth = base64.b64encode((sls.SALESFORCE_CLIENT_ID + ":" + sls.SALESFORCE_CLIENT_SECRET).encode("ascii"))\ + base64_auth = base64.b64encode((sls.SALESFORCE_CLIENT_ID + ":" + sls.SALESFORCE_CLIENT_SECRET).encode("ascii")) \ .decode("ascii") headers = { 'Content-Type': 'application/x-www-form-urlencoded', @@ -63,16 +65,16 @@ def add_user_to_crm(self, email, first_name, last_name, lang="en", educator=Fals language = "English" res = self.post(self.create_endpoint("Sefaria_App_User__c"), - json={ - "First_Name__c": first_name, - "Last_Name__c": last_name, - "Sefaria_App_Email__c": email, - "Hebrew_English__c": language, - "Educator__c": educator - }) + json={ + "First_Name__c": first_name, + "Last_Name__c": last_name, + "Sefaria_App_Email__c": email, + "Hebrew_English__c": language, + "Educator__c": educator + }) try: # add salesforce id to user profile - nationbuilder_user = res.json() # {'id': '1234', 'success': True, 'errors': []} + nationbuilder_user = res.json() # {'id': '1234', 'success': True, 'errors': []} return nationbuilder_user['id'] except: @@ -85,9 +87,9 @@ def change_user_email(self, uid, new_email): """ CrmConnectionManager.change_user_email(self, uid, new_email) res = self.patch(self.create_endpoint("Sefaria_App_User__c", uid), - json={ - "Sefaria_App_Email__c": new_email - }) + json={ + "Sefaria_App_Email__c": new_email + }) try: return res.status_code == 204 except: @@ -112,7 +114,8 @@ def find_crm_id(self, email=None): if email: CrmConnectionManager.validate_email(email) CrmConnectionManager.find_crm_id(self, email=email) - res = self.get(self.create_endpoint(f"query?=SELECT+id+FROM+Sefaria_App_User__c+WHERE+Sefaria_App_Email__c='{email}'")) + res = self.get( + self.create_endpoint(f"query?=SELECT+id+FROM+Sefaria_App_User__c+WHERE+Sefaria_App_Email__c='{email}'")) try: print(res) print(res.json()) @@ -129,13 +132,13 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en", language = "English" json_string = json.dumps({ - "Action": "Newsletter", - "First_Name__c": first_name, - "Last_Name__c": last_name, - "Sefaria_App_Email__c": email, - "Hebrew_English__c": language, - "Educator__c": educator - }) + "Action": "Newsletter", + "First_Name__c": first_name, + "Last_Name__c": last_name, + "Sefaria_App_Email__c": email, + "Hebrew_English__c": language, + "Educator__c": educator + }) res = self.post(self.create_endpoint("Sefaria_App_Data__c"), json={ "JSON_STRING__c": json_string @@ -154,6 +157,22 @@ def get_spam_users(self): def get_sustainers(self): pass + def create_job(self, operation, sobject): + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest" + body = json.dumps({ + "object": sobject, + "contentType": "CSV", + "operation": operation + }) + res = self.post(endpoint, data=body) + return res + + def find_job(self, operation, sobject): + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest" + res = self.get(endpoint) + job_id = list(filter(lambda x: x['operation'] == operation and x['object'] == 'sobject' and x['state'] == 'Open'))[0]['id'] + return job_id + def sf15to18(self, id): # from https://github.com/mslabina/sf15to18/blob/master/sf15to18.py if not id: @@ -183,5 +202,3 @@ def sf15to18(self, id): id += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ012345'[f] return id - - From d62c04b316dfa6a1226237eb976754a6e8d07252 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 22 Sep 2023 16:54:07 -0400 Subject: [PATCH 04/11] feat(salesforce): Implement get_sustainers [sc-20303] --- sefaria/helper/crm/salesforce.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/sefaria/helper/crm/salesforce.py b/sefaria/helper/crm/salesforce.py index 98ef5b6e41..167aa02302 100644 --- a/sefaria/helper/crm/salesforce.py +++ b/sefaria/helper/crm/salesforce.py @@ -155,7 +155,34 @@ def get_spam_users(self): return res def get_sustainers(self): - pass + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SUSTAINERS_REPORT}" + + data =None + while 1: + res = self.post(endpoint, json=data).json() + # verify sort + metadata = res['reportMetadata'] + columns = metadata['detailColumns'] + aggregates = metadata['aggregates'] + id_i = columns.index('CUST_ID') + active_sustainer_i = columns.index('FK_Contact.npsp__Sustainer__c') + rowcount_i = aggregates.index('RowCount') + # get index of ID + # get index of Sustainer + if res['factMap']['T!T']['aggregates'][rowcount_i]['value'] == 0: + break + + for row in res['factMap']['T!T']['rows']: + last_id = row['dataCells'][id_i] + yield last_id['value'] + data = {} + data['reportMetadata'] = res['reportMetadata'] + data["reportMetadata"]["reportFilters"].append({ + "value": last_id['value'], + "operator": "greaterThan", + "column": "CUST_ID" + }) + return res.json() def create_job(self, operation, sobject): endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest" From 6f5b35413779b51f3423863088e15bb41eddf5e1 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 22 Sep 2023 16:54:23 -0400 Subject: [PATCH 05/11] feat(salesforce): Add note to CRM mediator [sc-20303] --- sefaria/helper/crm/crm_mediator.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sefaria/helper/crm/crm_mediator.py b/sefaria/helper/crm/crm_mediator.py index ac73f224f6..3f524cde16 100644 --- a/sefaria/helper/crm/crm_mediator.py +++ b/sefaria/helper/crm/crm_mediator.py @@ -6,6 +6,13 @@ # todo: task queue, async class CrmMediator: + """ + Anywhere the *app* wants to perform CRM functions it should be using this, + rather than the other CRM classes. + + We do not want the CRM classes (i.e. salesforce.py) to be performing r/w operations outside + of r/w to the CRM. + """ def __init__(self): self._crm_connection = CrmFactory().get_connection_manager() From c60e07388f6adfafb26af76fa9b80d0ee590182c Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 22 Sep 2023 16:55:18 -0400 Subject: [PATCH 06/11] feat(salesforce-script): Update nation_builder_sync script so that sync actually gets run [sc-20303] --- scripts/nation_builder_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nation_builder_sync.py b/scripts/nation_builder_sync.py index e06b238f4c..f47d9299aa 100644 --- a/scripts/nation_builder_sync.py +++ b/scripts/nation_builder_sync.py @@ -35,8 +35,8 @@ gt = int(sys.argv[i][5:]) i+=1 - crm_mediator = CrmMediator() - crm_mediator.sync_sustainers() +crm_mediator = CrmMediator() +crm_mediator.sync_sustainers() # if sustainers_only: # connection_manager = CrmFactory().get_connection_manager() From f15e895aeb1eb884d6648f69d34e2922d6a99340 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 22 Sep 2023 16:57:34 -0400 Subject: [PATCH 07/11] refactor(salesforce-script): Allow purge_spammer_account_data to take an optional profile that is already found [sc-20303] --- sefaria/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sefaria/views.py b/sefaria/views.py index ab42bd7ff0..9fccc6c1e0 100644 --- a/sefaria/views.py +++ b/sefaria/views.py @@ -972,10 +972,11 @@ def delete_user_by_email(request): -def purge_spammer_account_data(spammer_id, delete_from_crm=True): +def purge_spammer_account_data(spammer_id, delete_from_crm=True, profile=None): from django.contrib.auth.models import User # Delete from Nationbuilder - profile = db.profiles.find_one({"id": spammer_id}) + if not profile: + profile = db.profiles.find_one({"id": spammer_id}) if delete_from_crm: try: crm_connection_manager = CrmMediator().get_connection_manager() From e32cbc81a8a7e970b4c45532d42224f86baeb67f Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Thu, 28 Sep 2023 16:53:57 -0400 Subject: [PATCH 08/11] fix(salesforce-script): Correctly find profile for Salesforce [sc-20303] --- sefaria/helper/crm/crm_info_store.py | 11 +++++++++-- sefaria/helper/crm/crm_mediator.py | 9 +++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/sefaria/helper/crm/crm_info_store.py b/sefaria/helper/crm/crm_info_store.py index cb46b41e1c..c8b6e62ca6 100644 --- a/sefaria/helper/crm/crm_info_store.py +++ b/sefaria/helper/crm/crm_info_store.py @@ -58,8 +58,15 @@ def get_current_sustainers(): @staticmethod def find_sustainer_profile(sustainer): - if sustainer['email']: - return UserProfile(email=sustainer['email']) + if sls.CRM_TYPE == "SALESFORCE": + try: + mongo_prof = db.profiles.find_one({"sf_app_user_id": sustainer}) + return UserProfile(id=mongo_prof['id']) + except: + return None + else: + if sustainer['email']: + return UserProfile(email=sustainer['email']) @staticmethod def mark_sustainer(profile, is_sustainer=True): diff --git a/sefaria/helper/crm/crm_mediator.py b/sefaria/helper/crm/crm_mediator.py index 3f524cde16..cc7afa6be2 100644 --- a/sefaria/helper/crm/crm_mediator.py +++ b/sefaria/helper/crm/crm_mediator.py @@ -34,10 +34,11 @@ def sync_sustainers(self): current_sustainers = CrmInfoStore.get_current_sustainers() for crm_sustainer in self._crm_connection.get_sustainers(): crm_sustainer_profile = CrmInfoStore.find_sustainer_profile(crm_sustainer) - if current_sustainers.get(crm_sustainer.id) is not None: # keep current sustainer - del current_sustainers[crm_sustainer.id] - else: - CrmInfoStore.mark_sustainer(crm_sustainer_profile) + if crm_sustainer_profile: # in case of out of date salesforce info + if current_sustainers.get(crm_sustainer.id) is not None: # keep current sustainer + del current_sustainers[crm_sustainer.id] + else: + CrmInfoStore.mark_sustainer(crm_sustainer_profile) for sustainer_to_remove in current_sustainers: CrmInfoStore.mark_sustainer(sustainer_to_remove, False) From d7a94a27f60868b944a7169619ed2dd441d71f55 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 29 Sep 2023 09:43:24 -0400 Subject: [PATCH 09/11] fix(salesforce-script): Fix find-job function and add comment for get_sustainers [sc-20303] --- sefaria/helper/crm/crm_info_store.py | 7 +++++++ sefaria/helper/crm/salesforce.py | 12 +++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sefaria/helper/crm/crm_info_store.py b/sefaria/helper/crm/crm_info_store.py index c8b6e62ca6..160bc5f547 100644 --- a/sefaria/helper/crm/crm_info_store.py +++ b/sefaria/helper/crm/crm_info_store.py @@ -52,6 +52,13 @@ def get_crm_id(uid=None, email=None, profile=None, crm_type=sls.CRM_TYPE): elif crm_type == "NONE": return True + @staticmethod + def get_by_crm_id(crm_id, crm_type=sls.CRM_TYPE): + if crm_type == "SALESFORCE": + return db.profiles.find_one({"sf_app_user_id": crm_id}) + else: + return None + @staticmethod def get_current_sustainers(): return {profile["id"]: profile for profile in db.profiles.find({"is_sustainer": True})} diff --git a/sefaria/helper/crm/salesforce.py b/sefaria/helper/crm/salesforce.py index 167aa02302..78a90722f3 100644 --- a/sefaria/helper/crm/salesforce.py +++ b/sefaria/helper/crm/salesforce.py @@ -152,12 +152,14 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en", def get_spam_users(self): endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SPAM_REPORT}" res = self.post(endpoint) - return res - + return res.json() + def get_sustainers(self): + """ + This function queries a report it expects to contain only active sustainers and returns their salesforce IDs + """ endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SUSTAINERS_REPORT}" - - data =None + data = None while 1: res = self.post(endpoint, json=data).json() # verify sort @@ -197,7 +199,7 @@ def create_job(self, operation, sobject): def find_job(self, operation, sobject): endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest" res = self.get(endpoint) - job_id = list(filter(lambda x: x['operation'] == operation and x['object'] == 'sobject' and x['state'] == 'Open'))[0]['id'] + job_id = list(filter(lambda x: x['operation'] == operation and x['object'] == 'sobject' and x['state'] == 'Open'), res.json())[0]['id'] return job_id def sf15to18(self, id): From 496c053952ce615f2b970db6b933eaa6d0c08799 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 29 Sep 2023 09:44:32 -0400 Subject: [PATCH 10/11] refactor(salesforce-script): look cleanerby using existing variable [sc-20303] --- sefaria/helper/crm/salesforce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/helper/crm/salesforce.py b/sefaria/helper/crm/salesforce.py index 78a90722f3..442aa6ac4f 100644 --- a/sefaria/helper/crm/salesforce.py +++ b/sefaria/helper/crm/salesforce.py @@ -178,7 +178,7 @@ def get_sustainers(self): last_id = row['dataCells'][id_i] yield last_id['value'] data = {} - data['reportMetadata'] = res['reportMetadata'] + data['reportMetadata'] = metadata data["reportMetadata"]["reportFilters"].append({ "value": last_id['value'], "operator": "greaterThan", From 1a3054d0b237023749b611c2ba3f6474ae99d733 Mon Sep 17 00:00:00 2001 From: Nissa Mai-Rose Date: Fri, 29 Sep 2023 09:55:57 -0400 Subject: [PATCH 11/11] refactor(salesforce-script): remove SALESFORCE_SPAM_REPORT variable --- .../sefaria-project/templates/configmap/local-settings-file.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml index 0098f0e8ff..ae5291dd8b 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml @@ -183,7 +183,6 @@ data: SALESFORCE_BASE_URL = os.getenv("SALESFORCE_BASE_URL") SALESFORCE_CLIENT_ID = os.getenv("SALESFORCE_CLIENT_ID") SALESFORCE_CLIENT_SECRET = os.getenv("SALESFORCE_CLIENT_SECRET") - SALESFORCE_SPAM_REPORT = os.getenv("SALESFORCE_SPAM_REPORT") SALESFORCE_SUSTAINERS_REPORT = os.getenv("SALESFORCE_SUSTAINERS_REPORT") DISABLE_INDEX_SAVE = False