From 0fd465df1320dbe96ab2302d4b1326fa21d0604b Mon Sep 17 00:00:00 2001 From: Ben Weissmann Date: Wed, 9 Dec 2020 12:26:17 -0500 Subject: [PATCH] Rework dev env --- Dockerfile | 19 ++++ Makefile | 20 ++++ README.md | 23 ++--- app.py | 23 ++--- db/init.sql | 9 +- db/load_data.sh | 6 +- docker-compose.yml | 12 ++- requirements.txt | 9 +- static/scripts.js | 1 + static/styles.css | 213 +++++++++++++++++++++++++++++++++++++++++++ templates/base.html | 218 +------------------------------------------- 11 files changed, 308 insertions(+), 245 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 static/scripts.js create mode 100644 static/styles.css diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f9e87b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.7 + +RUN pip install pipenv && \ + apt-get update && \ + apt-get install pv lsb-release -y && \ + sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \ + sh -c 'wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -' && \ + apt-get update && \ + apt-get -y install postgresql-client-12 + +RUN mkdir /app +WORKDIR /app + +ADD Pipfile /app/Pipfile +ADD Pipfile.lock /app/Pipfile.lock + +RUN pipenv install --dev + +CMD pipenv run serve diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9daa892 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +up: + docker-compose up --build + +shell: + docker-compose exec app bash + +pyshell: + docker-compose exec app ipython + +loaddata: + docker-compose exec app db/load_data.sh ${FILE} + +initsql: + docker-compose exec app bash -c "PGPASSWORD=postgres psql -h postgres -U postgres -d gatrack -f db/init.sql" + +lockdeps: + docker-compose exec app bash -c "pipenv lock --requirements > requirements.txt" + +format: + docker-compose exec app pipenv run format diff --git a/README.md b/README.md index 7267d3a..60bb686 100644 --- a/README.md +++ b/README.md @@ -2,24 +2,25 @@ ## Dev environment -To begin you'll need docker, pipenv, and python 3.7 installed locally to develop ga-track +To start up the dev environment, run `make up`. -1. To spin up a Postgres database run `docker-compose up` +The dev server will be running on `http://localhost:5050`. -2. Load the data into the postgres database: - 1. Create a copy of `.env.sample` named `.env`, modifying the parameters to connect to your database. - 2. Download the statewide zip files from https://elections.sos.ga.gov/Elections/voterabsenteefile.do for both the November general election (35209.zip) as well as the January runoff (35211.zip). - 3. Type `./db/load_data.sh 35209` to load the general election data, and `./db/load_data.sh 35211` to load the runoff data. (This script can also be used to update the data when a new version of the zip file becomes available.) - 4. Run the SQL commands in `db/init.sql` (this only needs to be done once, after the initial batch of data has been loaded for both elections). +### Loading data -3. In another shell, run `pipenv sync --dev` +1. Download the statewide zip files from https://elections.sos.ga.gov/Elections/voterabsenteefile.do for both the November general election (35209.zip) as well as the January runoff (35211.zip). Place these two zip files in the root of this repo. + +2. Run `make loaddata FILE=35209` and then `make loaddata FILE=35211`. Note that the first time you run these, you'll see a couple errors about "relation does not exist". That's OK. + +3. Run `make initsql`. + +To refresh the data, download and replace those two zip files, then run step 2 again (you don't have to run step 3 again). -4. To start the web app run `pipenv run serve` ## Extras -If you'd like to cleanup the python code formatting, run `pipenv run format` +If you'd like to cleanup the python code formatting, run `make format` ## Deploying to Heroku -Due to Heroku's poor support for Pipenv, if you change the requirements inside Pipfile you must run `pipenv lock --requirements > requirements.txt` +Due to Heroku's poor support for Pipenv, if you change the requirements inside Pipfile you must run `make lockdeps` to update `requirements.txt`. diff --git a/app.py b/app.py index 4924d3c..554af7c 100644 --- a/app.py +++ b/app.py @@ -2,12 +2,10 @@ import os import sentry_sdk -from flask import Flask, make_response, redirect, render_template, request -import jinja2 +from flask import Flask, make_response, render_template, request from sentry_sdk.integrations.flask import FlaskIntegration from analytics import statsd - from models import db from models.voters import VoteRecord @@ -30,10 +28,12 @@ app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False db.init_app(app) -@app.template_filter('commafy') + +@app.template_filter("commafy") def commafy_filter(v): return "{:,}".format(v) + def render_template_nocache(template_name, **args): resp = make_response(render_template(template_name, **args)) resp.headers.set("Cache-Control", "private,no-store") @@ -42,27 +42,28 @@ def render_template_nocache(template_name, **args): @app.route("/") def index(): - sql = ''' + sql = """ select (select count from voter_status_counters_35209 where "Application Status" = 'A' and "Ballot Status" = 'A') as returned_general, (select count from voter_status_counters_35211 - where "Application Status" = 'A' and "Ballot Status" = 'A') + where "Application Status" = 'A' and "Ballot Status" = 'A') as returned_special, (select count from voter_status_counters_35211 - where "Application Status" = 'A' and "Ballot Status" = 'total') + where "Application Status" = 'A' and "Ballot Status" = 'total') as applied_special, (select file_update_time from updated_times - where election = '35211' order by job_time desc limit 1) - as update_time''' + where election = '35211' order by job_time desc limit 1) + as update_time""" stats = db.engine.execute(sql).first() - resp = make_response(render_template("index.html", stats = stats)) + resp = make_response(render_template("index.html", stats=stats)) resp.headers.set("Cache-Control", "public, max-age=7200") return resp + @app.route("/faq") def faq(): resp = make_response(render_template("contact.html")) @@ -127,4 +128,4 @@ def pluralize(number, singular="", plural="s"): if __name__ == "__main__": - app.run(debug=DEBUG) + app.run(debug=DEBUG, host="0.0.0.0") diff --git a/db/init.sql b/db/init.sql index cfd9239..69aa34c 100644 --- a/db/init.sql +++ b/db/init.sql @@ -22,7 +22,7 @@ CREATE UNIQUE INDEX name_index ON all_voters ( CREATE INDEX voter_reg_index ON all_voters ("Voter Registration #"); CREATE OR REPLACE VIEW voters_and_statuses AS -SELECT v.*, +SELECT v.*, a."Application Status" as "Old App Status", a."Ballot Status" as "Old Ballot Status", a."Status Reason" as "Old Status Reason", a."Application Date" as "Old App Date", a."Ballot Issued Date" as "Old Issued Date", @@ -39,11 +39,11 @@ CREATE TABLE updated_times ( election text, job_time timestamp, file_update_time timestamp -) +); CREATE MATERIALIZED VIEW stats_by_county_day AS -SELECT "County" as county, days_before, +SELECT "County" as county, days_before, sum(("Application Status" = 'A' AND what = 'apply_general')::integer) as applied_general, sum(("Ballot Status" = 'A' AND @@ -92,3 +92,6 @@ ORDER BY days_before DESC; CREATE UNIQUE INDEX stats_by_county_index ON stats_by_county_day (county, days_before); CREATE INDEX stats_by_day_index ON stats_by_county_day (days_before); + +INSERT INTO updated_times VALUES ('35209', now(), now()); +INSERT INTO updated_times VALUES ('35211', now(), now()); diff --git a/db/load_data.sh b/db/load_data.sh index a091c5b..f2764a6 100755 --- a/db/load_data.sh +++ b/db/load_data.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Usage ./load_data.sh ELECTION_NUM # Assumes there is a file named "env" that sets a DB variable with the psql # connection string. and that there's a file ELECTION_NUM.zip which is the zip @@ -14,7 +14,7 @@ clean_up () { } trap clean_up EXIT -source ./.env +# source ./.env echo "Unzipping STATEWIDE.csv from $1.zip..." unzip $1.zip STATEWIDE.csv @@ -88,7 +88,7 @@ CREATE OR REPLACE VIEW voters_${ELECTION}_current AS SELECT * FROM $TABLE; -- this will (intentionally) fail if the table already exists -CREATE MATERIALIZED VIEW current_status_${ELECTION} AS +CREATE MATERIALIZED VIEW current_status_${ELECTION} AS SELECT DISTINCT ON("Voter Registration #") "County", "Voter Registration #", diff --git a/docker-compose.yml b/docker-compose.yml index 412ef62..65e242b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: postgres: image: postgres:alpine - restart: always + restart: unless-stopped environment: POSTGRES_DB: gatrack POSTGRES_PASSWORD: postgres @@ -15,6 +15,16 @@ services: - postgres:/var/lib/postgresql/data ports: - 5432:5432 + app: + build: . + restart: unless-stopped + environment: + DATABASE_URL: postgres://postgres:postgres@postgres:5432/gatrack + volumes: + - .:/app + ports: + - 5050:5000 + volumes: postgres: external: false diff --git a/requirements.txt b/requirements.txt index d358797..547cdcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,10 @@ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock --requirements +# + -i https://pypi.org/simple agate-dbf==0.2.2 agate-excel==0.2.3 @@ -37,7 +44,7 @@ pytimeparse==1.1.8 pytz==2020.4 requests==2.25.0 sentry-sdk==0.19.4 -six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' +six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' sortedcontainers==2.3.0 sqlalchemy==1.3.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' tenacity==6.2.0 diff --git a/static/scripts.js b/static/scripts.js new file mode 100644 index 0000000..f141d82 --- /dev/null +++ b/static/scripts.js @@ -0,0 +1 @@ +console.log("Scripts TODO"); diff --git a/static/styles.css b/static/styles.css new file mode 100644 index 0000000..43a476a --- /dev/null +++ b/static/styles.css @@ -0,0 +1,213 @@ +html, body { + height: 100%; +} + +body { + display: flex; + flex-direction: column; + + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; + -webkit-font-smoothing: antialiased; +} + +#navbar { + /* fixes navbar height bug in Safari */ + display: block; +} + +.navbar-container { + margin: 10px 0; +} + +.container.footer-container { + max-width: 1200px; +} + +@media only screen and (max-width: 600px) { + .navbar-container { + margin-left: 0; + margin-right: 0; + padding-left: 0; + padding-right: 0; + } + + #navbar .navbar-container { + flex-direction: column-reverse; + } + + #navbar .brand { + margin-bottom: 20px; + } +} + +#content { + flex: 1; +} + +#footer { + flex-shrink: 0; +} + +label { + font-weight: bold; +} + +.container { + max-width: 640px; + margin: auto; +} +.MuiSvgIcon-root { + width: 30px; + fill: black; +} +.icon-circle { + background-color: white; + border-radius: 50%; + padding: 8px; + margin: 8px; +} +.btn-danger { + background-color: #e12814; +} +.btn-primary { + background-color: #2274d8; +} +.card { + border-radius: 0; +} +#footer, #navbar { + background-color: black; + color: white; +} + +#footer { + color: rgb(192, 192, 192); +} + +#footer a { + color: rgb(192, 192, 192); + text-decoration: underline; +} + +#footer a:hover { + color:#2274d8; +} + +#footer nav a { + margin-left: 20px; + margin-right: 20px; +} + +#footer nav a:first-child { + margin-left: 0; +} + +#footer .btn { + color: white; + text-decoration: none; + font-size: 18px; + padding: 15px 0; +} + +#footer .btn:hover { + color: white; +} + +#footer .icon-circle { + transition: all 0.3s; + margin-top: 0; + margin-bottom: 30px; +} + +#footer .icon-circle:first-child { + transition: all 0.3s; + margin-left: 0; +} + +#footer .icon-circle:hover { + background-color: #2274d8; +} + +img.brand { + height: 40px; +} +.bg-dark, .navbar-dark { + background-color: black; +} +.checkbox { + color: #018845; + vertical-align: bottom; +} + +.list-group-item { + display: flex; + flex-direction: row; +} + +.list-icon { + width: 40px; + display: flex; + flex-direction: column; + justify-content: center; +} + +.list-text { + flex: 1; +} + +input { + text-transform: uppercase; +} + +.bottom-warning { + margin-top: 40px; +} + +::-webkit-input-placeholder { + text-transform: none; +} + +:-moz-placeholder { + text-transform: none; +} + +::-moz-placeholder { + text-transform: none; +} + +:-ms-input-placeholder { + text-transform: none; +} + +.material-icons { + font-family: 'Material Icons'; + font-weight: normal; + font-style: normal; + font-size: 24px; /* Preferred icon size */ + display: inline-block; + line-height: 1; + text-transform: none; + letter-spacing: normal; + word-wrap: normal; + white-space: nowrap; + direction: ltr; + + /* Support for all WebKit browsers. */ + -webkit-font-smoothing: antialiased; + /* Support for Safari and Chrome. */ + text-rendering: optimizeLegibility; + + /* Support for Firefox. */ + -moz-osx-font-smoothing: grayscale; + + /* Support for IE. */ + font-feature-settings: 'liga'; +} + +.brand a { + color: white; +} + +h3 { + margin-top: 40px; +} diff --git a/templates/base.html b/templates/base.html index 0bcda49..4b1babf 100644 --- a/templates/base.html +++ b/templates/base.html @@ -39,221 +39,7 @@ - + VoteAmerica Georgia Ballot Tracker {% endblock %} @@ -327,5 +113,7 @@

2021 Georgia Early Vote Ballot Tracker< + +