Feature/sc 20303/spam users are marked as spam in app then

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -183,6 +183,7 @@ data:
     SALESFORCE_BASE_URL = os.getenv("SALESFORCE_BASE_URL")
     SALESFORCE_CLIENT_ID = os.getenv("SALESFORCE_CLIENT_ID")
     SALESFORCE_CLIENT_SECRET = os.getenv("SALESFORCE_CLIENT_SECRET")
+    SALESFORCE_SUSTAINERS_REPORT = os.getenv("SALESFORCE_SUSTAINERS_REPORT")
 
     DISABLE_INDEX_SAVE = False
 

scripts/nation_builder_sync.py

@@ -35,8 +35,8 @@
         gt = int(sys.argv[i][5:])
     i+=1
 
-    crm_mediator = CrmMediator()
-    crm_mediator.sync_sustainers()
+crm_mediator = CrmMediator()
+crm_mediator.sync_sustainers()
 
 # if sustainers_only:
 #     connection_manager = CrmFactory().get_connection_manager()

sefaria/helper/crm/crm_info_store.py

@@ -52,14 +52,28 @@ def get_crm_id(uid=None, email=None, profile=None, crm_type=sls.CRM_TYPE):
         elif crm_type == "NONE":
             return True
 
+    @staticmethod
+    def get_by_crm_id(crm_id, crm_type=sls.CRM_TYPE):
+        if crm_type == "SALESFORCE":
+            return db.profiles.find_one({"sf_app_user_id": crm_id})
+        else:
+            return None
+
     @staticmethod
     def get_current_sustainers():
         return {profile["id"]: profile for profile in db.profiles.find({"is_sustainer": True})}
 
     @staticmethod
     def find_sustainer_profile(sustainer):
-        if sustainer['email']:
-            return UserProfile(email=sustainer['email'])
+        if sls.CRM_TYPE == "SALESFORCE":
+            try:
+                mongo_prof = db.profiles.find_one({"sf_app_user_id": sustainer})
+                return UserProfile(id=mongo_prof['id'])
+            except:
+                return None
+        else:
+            if sustainer['email']:
+                return UserProfile(email=sustainer['email'])
 
     @staticmethod
     def mark_sustainer(profile, is_sustainer=True):

sefaria/helper/crm/crm_mediator.py

@@ -6,6 +6,13 @@
 # todo: task queue, async
 
 class CrmMediator:
+    """
+    Anywhere the *app* wants to perform CRM functions it should be using this,
+    rather than the other CRM classes.
+
+    We do not want the CRM classes (i.e. salesforce.py) to be performing r/w operations outside
+    of r/w to the CRM.
+    """
     def __init__(self):
         self._crm_connection = CrmFactory().get_connection_manager()
 
@@ -27,10 +34,11 @@ def sync_sustainers(self):
         current_sustainers = CrmInfoStore.get_current_sustainers()
         for crm_sustainer in self._crm_connection.get_sustainers():
             crm_sustainer_profile = CrmInfoStore.find_sustainer_profile(crm_sustainer)
-            if current_sustainers.get(crm_sustainer.id) is not None:  # keep current sustainer
-                del current_sustainers[crm_sustainer.id]
-            else:
-                CrmInfoStore.mark_sustainer(crm_sustainer_profile)
+            if crm_sustainer_profile: # in case of out of date salesforce info
+                if current_sustainers.get(crm_sustainer.id) is not None:  # keep current sustainer
+                    del current_sustainers[crm_sustainer.id]
+                else:
+                    CrmInfoStore.mark_sustainer(crm_sustainer_profile)
 
         for sustainer_to_remove in current_sustainers:
             CrmInfoStore.mark_sustainer(sustainer_to_remove, False)

sefaria/helper/crm/salesforce.py

@@ -6,17 +6,19 @@
 from sefaria.helper.crm.crm_connection_manager import CrmConnectionManager
 from sefaria import settings as sls
 
+
 class SalesforceConnectionManager(CrmConnectionManager):
     def __init__(self):
         CrmConnectionManager.__init__(self, sls.SALESFORCE_BASE_URL)
         self.version = "56.0"
+        self.bulk_api_version = "54.0"
         self.resource_prefix = f"services/data/v{self.version}/sobjects/"
 
     def create_endpoint(self, *args):
         return f"{sls.SALESFORCE_BASE_URL}/{self.resource_prefix}{'/'.join(args)}"
 
     def make_request(self, request, **kwargs):
-        for attempt in range(0,3):
+        for attempt in range(0, 3):
             try:
                 return request(**kwargs).json()
             except Exception as e:
@@ -25,7 +27,7 @@ def make_request(self, request, **kwargs):
 
     def get(self, endpoint):
         headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
-        return self.session.get(endpoint, headers)
+        return self.session.get(endpoint, headers=headers)
 
     def post(self, endpoint, **kwargs):
         headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
@@ -37,7 +39,7 @@ def patch(self, endpoint, **kwargs):
 
     def _get_connection(self):
         access_token_url = "%s/services/oauth2/token?grant_type=client_credentials" % self.base_url
-        base64_auth = base64.b64encode((sls.SALESFORCE_CLIENT_ID + ":" + sls.SALESFORCE_CLIENT_SECRET).encode("ascii"))\
+        base64_auth = base64.b64encode((sls.SALESFORCE_CLIENT_ID + ":" + sls.SALESFORCE_CLIENT_SECRET).encode("ascii")) \
             .decode("ascii")
         headers = {
             'Content-Type': 'application/x-www-form-urlencoded',
@@ -63,16 +65,16 @@ def add_user_to_crm(self, email, first_name, last_name, lang="en", educator=Fals
             language = "English"
 
         res = self.post(self.create_endpoint("Sefaria_App_User__c"),
-                  json={
-                      "First_Name__c": first_name,
-                      "Last_Name__c": last_name,
-                      "Sefaria_App_Email__c": email,
-                      "Hebrew_English__c": language,
-                      "Educator__c": educator
-                  })
+                        json={
+                            "First_Name__c": first_name,
+                            "Last_Name__c": last_name,
+                            "Sefaria_App_Email__c": email,
+                            "Hebrew_English__c": language,
+                            "Educator__c": educator
+                        })
 
         try:  # add salesforce id to user profile
-            nationbuilder_user = res.json() # {'id': '1234', 'success': True, 'errors': []}
+            nationbuilder_user = res.json()  # {'id': '1234', 'success': True, 'errors': []}
             return nationbuilder_user['id']
 
         except:
@@ -85,9 +87,9 @@ def change_user_email(self, uid, new_email):
         """
         CrmConnectionManager.change_user_email(self, uid, new_email)
         res = self.patch(self.create_endpoint("Sefaria_App_User__c", uid),
-                 json={
-                     "Sefaria_App_Email__c": new_email
-                 })
+                         json={
+                             "Sefaria_App_Email__c": new_email
+                         })
         try:
             return res.status_code == 204
         except:
@@ -112,7 +114,8 @@ def find_crm_id(self, email=None):
         if email:
             CrmConnectionManager.validate_email(email)
         CrmConnectionManager.find_crm_id(self, email=email)
-        res = self.get(self.create_endpoint(f"query?=SELECT+id+FROM+Sefaria_App_User__c+WHERE+Sefaria_App_Email__c='{email}'"))
+        res = self.get(
+            self.create_endpoint(f"query?=SELECT+id+FROM+Sefaria_App_User__c+WHERE+Sefaria_App_Email__c='{email}'"))
         try:
             print(res)
             print(res.json())
@@ -129,13 +132,13 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en",
             language = "English"
 
         json_string = json.dumps({
-                      "Action": "Newsletter",
-                      "First_Name__c": first_name,
-                      "Last_Name__c": last_name,
-                      "Sefaria_App_Email__c": email,
-                      "Hebrew_English__c": language,
-                      "Educator__c": educator
-                  })
+            "Action": "Newsletter",
+            "First_Name__c": first_name,
+            "Last_Name__c": last_name,
+            "Sefaria_App_Email__c": email,
+            "Hebrew_English__c": language,
+            "Educator__c": educator
+        })
         res = self.post(self.create_endpoint("Sefaria_App_Data__c"),
                         json={
                             "JSON_STRING__c": json_string
@@ -145,3 +148,86 @@ def subscribe_to_lists(self, email, first_name=None, last_name=None, lang="en",
         except:
             return False
         return res
+
+    def get_spam_users(self):
+        endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SPAM_REPORT}"
+        res = self.post(endpoint)
+        return res.json()
+
+    def get_sustainers(self):
+        """
+        This function queries a report it expects to contain only active sustainers and returns their salesforce IDs
+        """
+        endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.version}/analytics/reports/{sls.SALESFORCE_SUSTAINERS_REPORT}"
+        data = None
+        while 1:
+            res = self.post(endpoint, json=data).json()
+            # verify sort
+            metadata = res['reportMetadata']
+            columns = metadata['detailColumns']
+            aggregates = metadata['aggregates']
+            id_i = columns.index('CUST_ID')
+            active_sustainer_i = columns.index('FK_Contact.npsp__Sustainer__c')
+            rowcount_i = aggregates.index('RowCount')
+            # get index of ID
+            # get index of Sustainer
+            if res['factMap']['T!T']['aggregates'][rowcount_i]['value'] == 0:
+                break
+
+            for row in res['factMap']['T!T']['rows']:
+                last_id = row['dataCells'][id_i]
+                yield last_id['value']
+            data = {}
+            data['reportMetadata'] = metadata
+            data["reportMetadata"]["reportFilters"].append({
+                    "value": last_id['value'],
+                    "operator": "greaterThan",
+                    "column": "CUST_ID"
+                })
+        return res.json()
+
+    def create_job(self, operation, sobject):
+        endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest"
+        body = json.dumps({
+            "object": sobject,
+            "contentType": "CSV",
+            "operation": operation
+        })
+        res = self.post(endpoint, data=body)
+        return res
+
+    def find_job(self, operation, sobject):
+        endpoint = f"{sls.SALESFORCE_BASE_URL}/services/data/v{self.bulk_api_version}/jobs/ingest"
+        res = self.get(endpoint)
+        job_id = list(filter(lambda x: x['operation'] == operation and x['object'] == 'sobject' and x['state'] == 'Open'), res.json())[0]['id']
+        return job_id
+
+    def sf15to18(self, id):
+        # from https://github.com/mslabina/sf15to18/blob/master/sf15to18.py
+        if not id:
+            raise ValueError('No id given.')
+        if not isinstance(id, str):
+            raise TypeError('The given id isn\'t a string')
+        if len(id) == 18:
+            return id
+        if len(id) != 15:
+            raise ValueError('The given id isn\'t 15 characters long.')
+
+        # Generate three last digits of the id
+        for i in range(0, 3):
+            f = 0
+
+            # For every 5-digit block of the given id
+            for j in range(0, 5):
+                # Assign the j-th chracter of the i-th 5-digit block to c
+                c = id[i * 5 + j]
+
+                # Check if c is an uppercase letter
+                if c >= 'A' and c <= 'Z':
+                    # Set a 1 at the character's position in the reversed segment
+                    f += 1 << j
+
+            # Add the calculated character for the current block to the id
+            id += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ012345'[f]
+
+        return id

sefaria/local_settings_example.py

@@ -191,6 +191,8 @@
 SALESFORCE_BASE_URL = ""
 SALESFORCE_CLIENT_ID = ""
 SALESFORCE_CLIENT_SECRET = ""
+SALESFORCE_SPAM_REPORT = ""
+SALESFORCE_SUSTAINERS_REPORT = ""
 
 # Issue bans to Varnish on update.
 USE_VARNISH = False

sefaria/views.py

@@ -972,10 +972,11 @@ def delete_user_by_email(request):
 
 
 
-def purge_spammer_account_data(spammer_id, delete_from_crm=True):
+def purge_spammer_account_data(spammer_id, delete_from_crm=True, profile=None):
     from django.contrib.auth.models import User
     # Delete from Nationbuilder
-    profile = db.profiles.find_one({"id": spammer_id})
+    if not profile:
+        profile = db.profiles.find_one({"id": spammer_id})
     if delete_from_crm:
         try:
             crm_connection_manager = CrmMediator().get_connection_manager()