new feature: subscriptions

Pipfile

@@ -29,6 +29,7 @@ ddtrace = "*"
 datadog = "*"
 sentry-sdk = "*"
 blinker = "*"
+flask-mail = "*"
 
 [requires]
 python_version = "3.7"
@@ -40,6 +41,7 @@ black = "black app.py"
 mypy = "mypy app.py --strict-optional"
 format = "bash -c 'pipenv run autoflake && pipenv run isort && pipenv run black'"
 serve = "python app.py"
+flask = "flask"
 
 [pipenv]
 allow_prereleases = true

app.py

@@ -1,15 +1,17 @@
-import logging
-import os
+import logging, os, smtplib
 
 import sentry_sdk
 from flask import Flask, make_response, redirect, render_template, request
+from flask_mail import Mail, Message
+from sqlalchemy.sql import func
 import jinja2
 from sentry_sdk.integrations.flask import FlaskIntegration
 
 from analytics import statsd
 
 from models import db
 from models.voters import VoteRecord
+from models.subscriptions import Subscription
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -29,6 +31,7 @@
 
 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
 db.init_app(app)
+mail = Mail(app)
 
 @app.template_filter('commafy')
 def commafy_filter(v):
@@ -70,6 +73,47 @@ def faq():
 
     return resp
 
+@app.route('/subscribe', methods=["POST"])
+def subscribe():
+    voter_reg_num = request.values.get("voter_reg_num")
+    email = request.values.get("email")
+
+    new_sub = Subscription(voter_reg_num=voter_reg_num,
+                           email=email)
+    db.session.add(new_sub)
+    db.session.commit()
+
+    return 'success'
+
+@app.route('/unsubscribe')
+def unsubscribe():
+    sub_id = request.values.get("sub_id")
+    secret = request.values.get("s")
+
+    if secret != Subscription.secret(sub_id):
+        return f"Invalid parameter: s={secret}, please make sure you clicked the correct link"
+
+    sub = Subscription.query.get(sub_id)
+    sub.active = False
+    db.session.commit()
+
+    return f"We will stop sending emails to {sub.email} about that voter."
+
+@app.route('/unsubscribe_all')
+def unsubscribe_all():
+    email = request.values.get("email")
+    secret = request.values.get("s")
+
+    if secret != Subscription.secret(email):
+        return f"Invalid parameter: s={s}, please make sure you clicked the correct link"
+
+    (Subscription.query
+                 .filter(Subscription.email == email)
+                 .update({Subscription.active: False}))
+    db.session.commit()
+
+    return (f"We will stop sending any emails to {email}.")
+
 
 @app.route("/search", methods=["GET"])
 def search():
@@ -125,6 +169,51 @@ def pluralize(number, singular="", plural="s"):
     else:
         return plural
 
+DAYS_BETWEEN_EMAILS = 7
+
+def generate_digest_email(address):
+    subscriptions = (Subscription.query
+        .filter(Subscription.active)
+        .filter(Subscription.email == address))
+    data = []
+    unsub_all = ''
+    for s in subscriptions:
+        if s.voter_reg_num:
+            voter = VoteRecord.query.get(s.voter_reg_num)
+        else:
+            # TODO: gather data about this search...
+            pass
+        data.append((voter, s.unsub_url(False)))
+
+        s.last_emailed = func.now()
+        db.session.add(s)
+        # just for convenience, we generate it each time, but it will be shared
+        # across all of them
+        unsub_all = s.unsub_url(True) 
+    return Message(
+        recipients=[address],
+        subject='Ballot update for your friends in Georgia',
+        html=render_template('digest-email.html', email=address, data=data, unsub_all=unsub_all)
+    )
+
+@app.cli.command('send-emails')
+def send_emails():
+    result = db.engine.execute(f'''
+        SELECT DISTINCT(email) FROM subscriptions
+        WHERE active AND
+            extract(epoch FROM
+                (now() - greatest(last_emailed, subscribe_time))
+            )/3600 > {24*DAYS_BETWEEN_EMAILS}
+    ''')
+    with mail.connect() as conn:
+        for (email,) in result:
+            message = generate_digest_email(email)
+            try:
+                conn.send(message)
+                db.session.commit()
+            except smtplib.SMTPException as e:
+                logging.warning(f'Exception {e} while sending an email to "{email}"')
+                db.session.rollback()
 
 if __name__ == "__main__":
     app.run(debug=DEBUG)

config.py

@@ -10,6 +10,13 @@ class Config(object):
         SQLALCHEMY_DATABASE_URI = os.environ['DATABASE_URL']
     else:
         SQLALCHEMY_DATABASE_URI = "postgres://postgres:postgres@localhost:5432/gatrack"
+    MAIL_SERVER='smtp.sendgrid.net'
+    MAIL_PORT=587
+    MAIL_USE_TLS=True
+    MAIL_USERNAME='apikey'
+    MAIL_PASSWORD=os.environ['SENDGRID_API_KEY']
+    MAIL_DEFAULT_SENDER='[email protected]'
+
 
 class ProductionConfig(Config):
     DEBUG = False

db/init.sql

@@ -39,4 +39,23 @@ CREATE TABLE updated_times (
   election text,
   job_time timestamp,
   file_update_time timestamp
-)
+);
+
+CREATE TABLE subscriptions (
+  id serial primary key,
+  email text,
+  voter_reg_num integer,
+
+  -- this is in case we want to enable people to subscribe to a full search
+  -- rather than an individual voter (in case their friend is not yet
+  -- showing up in the db at all):
+  search_params jsonb,
+
+  -- whether or not the subscription is active (will come in handy if we send an
+  -- email with an unsubscribe link!)
+  active boolean,
+
+  subscribe_time timestamp not null default now(),
+  last_emailed timestamp
+);
+CREATE INDEX ON subscriptions (email);

models/subscriptions.py

@@ -0,0 +1,35 @@
+import hashlib
+import os
+
+from models import db
+from sqlalchemy.sql import func
+
+class Subscription(db.Model):
+    __tablename__ = "subscriptions"
+
+    id = db.Column("id", db.Integer, primary_key=True)
+    email = db.Column("email", db.String())
+    voter_reg_num = db.Column("voter_reg_num", db.Integer())
+    active = db.Column("active", db.Boolean(), default=True)
+    search_params = db.Column("search_params", db.JSON())
+    subscribe_time = db.Column("subscribe_time", db.DateTime(), default=func.now())
+    last_emailed = db.Column("last_emailed", db.DateTime())
+
+    @staticmethod
+    def secret(data):
+        m = hashlib.sha256()
+        m.update(os.environ.get('UNSUBSCRIBE_KEY', 'top secret').encode())
+        m.update(f'{data}'.encode())
+        return m.hexdigest()
+
+    def unsub_url(self, all):
+        url = 'https://gaballot.org/unsubscribe'
+        if all:
+            url += '_all?email='
+            data = self.email
+        else:
+            url += '?sub_id='
+            data = self.id
+        url += f'{data}&s={Subscription.secret(data)}'
+        return url
+

requirements.txt

@@ -1,3 +1,10 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
 -i https://pypi.org/simple
 agate-dbf==0.2.2
 agate-excel==0.2.3
@@ -15,6 +22,7 @@ dbfread==2.0.7
 ddtrace==0.44.0
 decorator==4.4.2
 et-xmlfile==1.0.1
+flask-mail==0.9.1
 flask-restful==0.3.8
 flask-sqlalchemy==2.4.4
 flask==1.1.2
@@ -37,7 +45,7 @@ pytimeparse==1.1.8
 pytz==2020.4
 requests==2.25.0
 sentry-sdk==0.19.4
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sortedcontainers==2.3.0
 sqlalchemy==1.3.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 tenacity==6.2.0

templates/base.html

@@ -197,7 +197,7 @@
         flex: 1;
       }
 
-      input {
+      input.capitalized {
         text-transform: uppercase;
       }
 

templates/digest-email.html

@@ -0,0 +1,19 @@
+Here's an update on your friends in Georgia:
+
+<ul class="list-group list-group-flush">
+  {% for record in data %}
+    <li class="list-group-item">
+      <div class="list-text">
+        <p>
+          {{record[0].first}} {{ record[0].middle or '' }} {{record[0].last}}<br />
+          {{ record[0].city }}, {{record[0].county}} COUNTY
+        </p>
+        <p>{{ record[0].friendly_ballot_status(true) }}</p>
+        <p><a href="{{ record[1] }}">Stop getting emails about this person</a></p>
+      </div>
+    </li>
+  {% endfor %}
+</ul>
+You are receiving this because your email address was entered at
+gaballot.org. If you no longer would like to receive these emails, you can
+<a href="{{ unsub_all }}">unsubscribe</a>.

templates/index.html

@@ -18,11 +18,11 @@ <h2 class="h4">Search by name</h2>
     <div class="form-row">
       <div class="col-md-6 col-sm-12 mb-3">
         <label for="input-first">First / given name</label>
-        <input type="text" class="form-control" placeholder="Given name" name="first" id="input-first" required aria-required="true">
+        <input type="text" class="form-control capitalized" placeholder="Given name" name="first" id="input-first" required aria-required="true">
       </div>
       <div class="col-md-6 col-sm-12 mb-3">
         <label for="input-last">Last / family name</label>
-        <input type="text" class="form-control" placeholder="Family name" name="last" id="input-last" required aria-required="true">
+        <input type="text" class="form-control capitalized" placeholder="Family name" name="last" id="input-last" required aria-required="true">
       </div>
     </div>
     <div class="form-row">
@@ -33,15 +33,15 @@ <h2 class="h4">Search by name</h2>
     <div class="form-row">
       <div class="col-md-4 col-sm-12 mb-3">
         <label for="input-first">Middle name/initial</label>
-        <input type="text" class="form-control" placeholder="Middle name" name="middle" id="input-middle">
+        <input type="text" class="form-control capitalized" placeholder="Middle name" name="middle" id="input-middle">
       </div>
       <div class="col-md-4 col-sm-12 mb-3">
         <label for="input-last">County</label>
-        <input type="text" class="form-control" placeholder="County" name="county" id="input-county">
+        <input type="text" class="form-control capitalized" placeholder="County" name="county" id="input-county">
       </div>
       <div class="col-md-4 col-sm-12 mb-3">
         <label for="input-last">City</label>
-        <input type="text" class="form-control" placeholder="City" name="city" id="input-city">
+        <input type="text" class="form-control capitalized" placeholder="City" name="city" id="input-city">
       </div>
     </div>
     <div class="form-row">