Rework dev env

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.7
+
+RUN pip install pipenv && \
+  apt-get update && \
+  apt-get install pv lsb-release -y && \
+  sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
+  sh -c 'wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -' && \
+  apt-get update && \
+  apt-get -y install postgresql-client-12
+
+RUN mkdir /app
+WORKDIR /app
+
+ADD Pipfile /app/Pipfile
+ADD Pipfile.lock /app/Pipfile.lock
+
+RUN pipenv install --dev
+
+CMD pipenv run serve

Makefile

@@ -0,0 +1,20 @@
+up:
+	docker-compose up --build
+
+shell:
+	docker-compose exec app bash
+
+pyshell:
+	docker-compose exec app ipython
+
+loaddata:
+	docker-compose exec app db/load_data.sh ${FILE}
+
+initsql:
+	docker-compose exec app bash -c "PGPASSWORD=postgres psql -h postgres -U postgres -d gatrack -f db/init.sql"
+
+lockdeps:
+	docker-compose exec app bash -c "pipenv lock --requirements > requirements.txt"
+
+format:
+	docker-compose exec app pipenv run format

README.md

@@ -2,24 +2,25 @@
 
 ## Dev environment
 
-To begin you'll need docker, pipenv, and python 3.7 installed locally to develop ga-track
+To start up the dev environment, run `make up`.
 
-1. To spin up a Postgres database run `docker-compose up`
+The dev server will be running on `http://localhost:5050`.
 
-2. Load the data into the postgres database:
-    1. Create a copy of `.env.sample` named `.env`, modifying the parameters to connect to your database.
-    2. Download the statewide zip files from https://elections.sos.ga.gov/Elections/voterabsenteefile.do for both the November general election (35209.zip) as well as the January runoff (35211.zip).
-    3. Type `./db/load_data.sh 35209` to load the general election data, and `./db/load_data.sh 35211` to load the runoff data. (This script can also be used to update the data when a new version of the zip file becomes available.)
-    4. Run the SQL commands in `db/init.sql` (this only needs to be done once, after the initial batch of data has been loaded for both elections).
+### Loading data
 
-3. In another shell, run `pipenv sync --dev`
+1. Download the statewide zip files from https://elections.sos.ga.gov/Elections/voterabsenteefile.do for both the November general election (35209.zip) as well as the January runoff (35211.zip). Place these two zip files in the root of this repo.
+
+2. Run `make loaddata FILE=35209` and then `make loaddata FILE=35211`. Note that the first time you run these, you'll see a couple errors about "relation does not exist". That's OK.
+
+3. Run `make initsql`.
+
+To refresh the data, download and replace those two zip files, then run step 2 again (you don't have to run step 3 again).
 
-4. To start the web app run `pipenv run serve`
 
 ## Extras
 
-If you'd like to cleanup the python code formatting, run `pipenv run format`
+If you'd like to cleanup the python code formatting, run `make format`
 
 ## Deploying to Heroku
 
-Due to Heroku's poor support for Pipenv, if you change the requirements inside Pipfile you must run `pipenv lock --requirements > requirements.txt`
+Due to Heroku's poor support for Pipenv, if you change the requirements inside Pipfile you must run `make lockdeps` to update `requirements.txt`.

app.py

@@ -2,12 +2,10 @@
 import os
 
 import sentry_sdk
-from flask import Flask, make_response, redirect, render_template, request
-import jinja2
+from flask import Flask, make_response, render_template, request
 from sentry_sdk.integrations.flask import FlaskIntegration
 
 from analytics import statsd
-
 from models import db
 from models.voters import VoteRecord
 
@@ -30,10 +28,12 @@
 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
 db.init_app(app)
 
-@app.template_filter('commafy')
+
+@app.template_filter("commafy")
 def commafy_filter(v):
     return "{:,}".format(v)
 
+
 def render_template_nocache(template_name, **args):
     resp = make_response(render_template(template_name, **args))
     resp.headers.set("Cache-Control", "private,no-store")
@@ -42,27 +42,28 @@ def render_template_nocache(template_name, **args):
 
 @app.route("/")
 def index():
-    sql = '''
+    sql = """
 select
     (select count from voter_status_counters_35209
      where "Application Status" = 'A' and "Ballot Status" = 'A')
         as returned_general,
     (select count from voter_status_counters_35211
-     where "Application Status" = 'A' and "Ballot Status" = 'A') 
+     where "Application Status" = 'A' and "Ballot Status" = 'A')
         as returned_special,
     (select count from voter_status_counters_35211
-     where "Application Status" = 'A' and "Ballot Status" = 'total') 
+     where "Application Status" = 'A' and "Ballot Status" = 'total')
         as applied_special,
     (select file_update_time from updated_times
-     where election = '35211' order by job_time desc limit 1) 
-        as update_time'''
+     where election = '35211' order by job_time desc limit 1)
+        as update_time"""
     stats = db.engine.execute(sql).first()
 
-    resp = make_response(render_template("index.html", stats = stats))
+    resp = make_response(render_template("index.html", stats=stats))
     resp.headers.set("Cache-Control", "public, max-age=7200")
 
     return resp
 
+
 @app.route("/faq")
 def faq():
     resp = make_response(render_template("contact.html"))
@@ -127,4 +128,4 @@ def pluralize(number, singular="", plural="s"):
 
 
 if __name__ == "__main__":
-    app.run(debug=DEBUG)
+    app.run(debug=DEBUG, host="0.0.0.0")

db/init.sql

@@ -22,7 +22,7 @@ CREATE UNIQUE INDEX name_index ON all_voters (
 CREATE INDEX voter_reg_index ON all_voters ("Voter Registration #");
 
 CREATE OR REPLACE VIEW voters_and_statuses AS
-SELECT v.*, 
+SELECT v.*,
          a."Application Status" as "Old App Status", a."Ballot Status" as "Old Ballot Status",
          a."Status Reason" as "Old Status Reason",
          a."Application Date" as "Old App Date", a."Ballot Issued Date" as "Old Issued Date",
@@ -39,11 +39,11 @@ CREATE TABLE updated_times (
   election text,
   job_time timestamp,
   file_update_time timestamp
-)
+);
 
 
 CREATE MATERIALIZED VIEW stats_by_county_day AS
-SELECT "County" as county, days_before, 
+SELECT "County" as county, days_before,
   sum(("Application Status" = 'A' AND
         what = 'apply_general')::integer) as applied_general,
   sum(("Ballot Status" = 'A' AND
@@ -92,3 +92,6 @@ ORDER BY days_before DESC;
 
 CREATE UNIQUE INDEX stats_by_county_index ON stats_by_county_day (county, days_before);
 CREATE INDEX stats_by_day_index ON stats_by_county_day (days_before);
+
+INSERT INTO updated_times VALUES ('35209', now(), now());
+INSERT INTO updated_times VALUES ('35211', now(), now());

db/load_data.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Usage ./load_data.sh ELECTION_NUM
 # Assumes there is a file named "env" that sets a DB variable with the psql
 # connection string. and that there's a file ELECTION_NUM.zip which is the zip
@@ -14,7 +14,7 @@ clean_up () {
 }
 trap clean_up EXIT
 
-source ./.env
+# source ./.env
 
 echo "Unzipping STATEWIDE.csv from $1.zip..."
 unzip $1.zip STATEWIDE.csv
@@ -88,7 +88,7 @@ CREATE OR REPLACE VIEW voters_${ELECTION}_current AS
   SELECT * FROM $TABLE;
 
 -- this will (intentionally) fail if the table already exists
-CREATE MATERIALIZED VIEW current_status_${ELECTION} AS 
+CREATE MATERIALIZED VIEW current_status_${ELECTION} AS
   SELECT DISTINCT ON("Voter Registration #")
     "County",
     "Voter Registration #",

docker-compose.yml

@@ -6,7 +6,7 @@ version: '3'
 services:
   postgres:
     image: postgres:alpine
-    restart: always
+    restart: unless-stopped
     environment:
       POSTGRES_DB: gatrack
       POSTGRES_PASSWORD: postgres
@@ -15,6 +15,16 @@ services:
       - postgres:/var/lib/postgresql/data
     ports:
       - 5432:5432
+  app:
+    build: .
+    restart: unless-stopped
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@postgres:5432/gatrack
+    volumes:
+      - .:/app
+    ports:
+      - 5050:5000
+
 volumes:
   postgres:
     external: false

requirements.txt

@@ -1,3 +1,10 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
 -i https://pypi.org/simple
 agate-dbf==0.2.2
 agate-excel==0.2.3
@@ -37,7 +44,7 @@ pytimeparse==1.1.8
 pytz==2020.4
 requests==2.25.0
 sentry-sdk==0.19.4
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sortedcontainers==2.3.0
 sqlalchemy==1.3.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 tenacity==6.2.0

static/scripts.js

@@ -0,0 +1 @@
+console.log("Scripts TODO");