diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml index a3b00fbe4b..ae5291dd8b 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml @@ -183,6 +183,7 @@ data: SALESFORCE_BASE_URL = os.getenv("SALESFORCE_BASE_URL") SALESFORCE_CLIENT_ID = os.getenv("SALESFORCE_CLIENT_ID") SALESFORCE_CLIENT_SECRET = os.getenv("SALESFORCE_CLIENT_SECRET") + SALESFORCE_SUSTAINERS_REPORT = os.getenv("SALESFORCE_SUSTAINERS_REPORT") DISABLE_INDEX_SAVE = False diff --git a/scripts/nation_builder_sync.py b/scripts/nation_builder_sync.py index e06b238f4c..f47d9299aa 100644 --- a/scripts/nation_builder_sync.py +++ b/scripts/nation_builder_sync.py @@ -35,8 +35,8 @@ gt = int(sys.argv[i][5:]) i+=1 - crm_mediator = CrmMediator() - crm_mediator.sync_sustainers() +crm_mediator = CrmMediator() +crm_mediator.sync_sustainers() # if sustainers_only: # connection_manager = CrmFactory().get_connection_manager() diff --git a/sefaria/helper/crm/crm_info_store.py b/sefaria/helper/crm/crm_info_store.py index cb46b41e1c..160bc5f547 100644 --- a/sefaria/helper/crm/crm_info_store.py +++ b/sefaria/helper/crm/crm_info_store.py @@ -52,14 +52,28 @@ def get_crm_id(uid=None, email=None, profile=None, crm_type=sls.CRM_TYPE): elif crm_type == "NONE": return True + @staticmethod + def get_by_crm_id(crm_id, crm_type=sls.CRM_TYPE): + if crm_type == "SALESFORCE": + return db.profiles.find_one({"sf_app_user_id": crm_id}) + else: + return None + @staticmethod def get_current_sustainers(): return {profile["id"]: profile for profile in db.profiles.find({"is_sustainer": True})} @staticmethod def find_sustainer_profile(sustainer): - if sustainer['email']: - return UserProfile(email=sustainer['email']) + if sls.CRM_TYPE == "SALESFORCE": + try: + mongo_prof = db.profiles.find_one({"sf_app_user_id": sustainer}) + return UserProfile(id=mongo_prof['id']) + except: + return None + else: + if sustainer['email']: + return UserProfile(email=sustainer['email']) @staticmethod def mark_sustainer(profile, is_sustainer=True): diff --git a/sefaria/helper/crm/crm_mediator.py b/sefaria/helper/crm/crm_mediator.py index ac73f224f6..cc7afa6be2 100644 --- a/sefaria/helper/crm/crm_mediator.py +++ b/sefaria/helper/crm/crm_mediator.py @@ -6,6 +6,13 @@ # todo: task queue, async class CrmMediator: + """ + Anywhere the *app* wants to perform CRM functions it should be using this, + rather than the other CRM classes. + + We do not want the CRM classes (i.e. salesforce.py) to be performing r/w operations outside + of r/w to the CRM. + """ def __init__(self): self._crm_connection = CrmFactory().get_connection_manager() @@ -27,10 +34,11 @@ def sync_sustainers(self): current_sustainers = CrmInfoStore.get_current_sustainers() for crm_sustainer in self._crm_connection.get_sustainers(): crm_sustainer_profile = CrmInfoStore.find_sustainer_profile(crm_sustainer) - if current_sustainers.get(crm_sustainer.id) is not None: # keep current sustainer - del current_sustainers[crm_sustainer.id] - else: - CrmInfoStore.mark_sustainer(crm_sustainer_profile) + if crm_sustainer_profile: # in case of out of date salesforce info + if current_sustainers.get(crm_sustainer.id) is not None: # keep current sustainer + del current_sustainers[crm_sustainer.id] + else: + CrmInfoStore.mark_sustainer(crm_sustainer_profile) for sustainer_to_remove in current_sustainers: CrmInfoStore.mark_sustainer(sustainer_to_remove, False) diff --git a/sefaria/helper/crm/salesforce.py b/sefaria/helper/crm/salesforce.py index 8f850d803f..442aa6ac4f 100644 --- a/sefaria/helper/crm/salesforce.py +++ b/sefaria/helper/crm/salesforce.py @@ -6,17 +6,19 @@ from sefaria.helper.crm.crm_connection_manager import CrmConnectionManager from sefaria import settings as sls + class SalesforceConnectionManager(CrmConnectionManager): def __init__(self): CrmConnectionManager.__init__(self, sls.SALESFORCE_BASE_URL) self.version = "56.0" + self.bulk_api_version = "54.0" self.resource_prefix = f"services/data/v{self.version}/sobjects/" def create_endpoint(self, *args): return f"{sls.SALESFORCE_BASE_URL}/{self.resource_prefix}{'/'.join(args)}" def make_request(self, request, **kwargs): - for attempt in range(0,3): + for attempt in range(0, 3): try: return request(**kwargs).json() except Exception as e: @@ -25,7 +27,7 @@ def make_request(self, request, **kwargs): def get(self, endpoint): headers = {'Content-type': 'application/json', 'Accept': 'application/json'} - return self.session.get(endpoint, headers) + return self.session.get(endpoint, headers=headers) def post(self, endpoint, **kwargs): headers = {'Content-type': 'application/json', 'Accept': 'application/json'} @@ -37,7 +39,7 @@ def patch(self, endpoint, **kwargs): def _get_connection(self): access_token_url = "%s/services/oauth2/token?grant_type=client_credentials" % self.base_url - base64_auth = base64.b64encode((sls.SALESFORCE_CLIENT_ID + ":" + sls.SALESFORCE_CLIENT_SECRET).encode("ascii"))\ + base64_auth = base64.b64encode((sls.SALESFORCE_CLIENT_ID + ":" + sls.SALESFORCE_CLIENT_SECRET).encode("ascii")) \ .decode("ascii") headers = { 'Content-Type': 'application/x-www-form-urlencoded', @@ -63,16 +65,16 @@ def add_user_to_crm(self, email, first_name, last_name, lang="en", educator=Fals language = "English" res = self.post(self.create_endpoint("Sefaria_App_User__c"), - json={ - "First_Name__c": first_name, - "Last_Name__c": last_name, - "Sefaria_App_Email__c": email, - "Hebrew_English__c": language, - "Educator__c": educator - }) + json={ + "First_Name__c": first_name, + "Last_Name__c": last_name, + "Sefaria_App_Email__c": email, + "Hebrew_English__c": language, + "Educator__c": educator + }) try: # add salesforce id to user profile - nationbuilder_user = res.json() # {'id': '1234', 'success': True, 'errors': []} + nationbuilder_user = res.json() # {'id': '1234', 'success': True, 'errors': []} return nationbuilder_user['id'] except: @@ -85,9 +87,9 @@ def change_user_email(self, uid, new_email): """ CrmConnectionManager.change_user_email(self, uid, new_email) res = self.patch(self.create_endpoint("Sefaria_App_User__c", uid), - json={ - "Sefaria_App_Email__c": new_email - }) + json={ + "Sefaria_App_Email__c": new_email + }) try: return res.status_code == 204 except: @@ -112,7 +114,8 @@ def find_crm_id(self, email=None): if email: CrmConnectionManager.validate_email(email) CrmConnectionManager.find_crm_id(self, email=email) - res = self.get(self.create_endpoint(f"query?=SELECT+id+FROM+Sefaria_App_User__c+WHERE+Sefaria_App_Email__c='{email}'")) + res = self.get( + self.create_endpoint(f"query?=SELECT+id+FROM+Sefaria_App_User__c+WHERE+Sefaria_App_Email__c='{email}'")) try: print(res) print(res.json()) @@ -129,13 +132,13 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en", language = "English" json_string = json.dumps({ - "Action": "Newsletter", - "First_Name__c": first_name, - "Last_Name__c": last_name, - "Sefaria_App_Email__c": email, - "Hebrew_English__c": language, - "Educator__c": educator - }) + "Action": "Newsletter", + "First_Name__c": first_name, + "Last_Name__c": last_name, + "Sefaria_App_Email__c": email, + "Hebrew_English__c": language, + "Educator__c": educator + }) res = self.post(self.create_endpoint("Sefaria_App_Data__c"), json={ "JSON_STRING__c": json_string @@ -145,3 +148,86 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en", except: return False return res + + def get_spam_users(self): + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SPAM_REPORT}" + res = self.post(endpoint) + return res.json() + + def get_sustainers(self): + """ + This function queries a report it expects to contain only active sustainers and returns their salesforce IDs + """ + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SUSTAINERS_REPORT}" + data = None + while 1: + res = self.post(endpoint, json=data).json() + # verify sort + metadata = res['reportMetadata'] + columns = metadata['detailColumns'] + aggregates = metadata['aggregates'] + id_i = columns.index('CUST_ID') + active_sustainer_i = columns.index('FK_Contact.npsp__Sustainer__c') + rowcount_i = aggregates.index('RowCount') + # get index of ID + # get index of Sustainer + if res['factMap']['T!T']['aggregates'][rowcount_i]['value'] == 0: + break + + for row in res['factMap']['T!T']['rows']: + last_id = row['dataCells'][id_i] + yield last_id['value'] + data = {} + data['reportMetadata'] = metadata + data["reportMetadata"]["reportFilters"].append({ + "value": last_id['value'], + "operator": "greaterThan", + "column": "CUST_ID" + }) + return res.json() + + def create_job(self, operation, sobject): + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest" + body = json.dumps({ + "object": sobject, + "contentType": "CSV", + "operation": operation + }) + res = self.post(endpoint, data=body) + return res + + def find_job(self, operation, sobject): + endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest" + res = self.get(endpoint) + job_id = list(filter(lambda x: x['operation'] == operation and x['object'] == 'sobject' and x['state'] == 'Open'), res.json())[0]['id'] + return job_id + + def sf15to18(self, id): + # from https://github.com/mslabina/sf15to18/blob/master/sf15to18.py + if not id: + raise ValueError('No id given.') + if not isinstance(id, str): + raise TypeError('The given id isn\'t a string') + if len(id) == 18: + return id + if len(id) != 15: + raise ValueError('The given id isn\'t 15 characters long.') + + # Generate three last digits of the id + for i in range(0, 3): + f = 0 + + # For every 5-digit block of the given id + for j in range(0, 5): + # Assign the j-th chracter of the i-th 5-digit block to c + c = id[i * 5 + j] + + # Check if c is an uppercase letter + if c >= 'A' and c <= 'Z': + # Set a 1 at the character's position in the reversed segment + f += 1 << j + + # Add the calculated character for the current block to the id + id += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ012345'[f] + + return id diff --git a/sefaria/local_settings_example.py b/sefaria/local_settings_example.py index f6b1fbf201..9f1d700800 100644 --- a/sefaria/local_settings_example.py +++ b/sefaria/local_settings_example.py @@ -191,6 +191,8 @@ SALESFORCE_BASE_URL = "" SALESFORCE_CLIENT_ID = "" SALESFORCE_CLIENT_SECRET = "" +SALESFORCE_SPAM_REPORT = "" +SALESFORCE_SUSTAINERS_REPORT = "" # Issue bans to Varnish on update. USE_VARNISH = False diff --git a/sefaria/views.py b/sefaria/views.py index ab42bd7ff0..9fccc6c1e0 100644 --- a/sefaria/views.py +++ b/sefaria/views.py @@ -972,10 +972,11 @@ def delete_user_by_email(request): -def purge_spammer_account_data(spammer_id, delete_from_crm=True): +def purge_spammer_account_data(spammer_id, delete_from_crm=True, profile=None): from django.contrib.auth.models import User # Delete from Nationbuilder - profile = db.profiles.find_one({"id": spammer_id}) + if not profile: + profile = db.profiles.find_one({"id": spammer_id}) if delete_from_crm: try: crm_connection_manager = CrmMediator().get_connection_manager()