diff --git a/README.md b/README.md index c0781ea..0cb94d8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,20 @@ If you are a PG&E customer you can link your account now! If you are not a PG&E # Development +## Environment Setup (Docker) + +You can get an environment setup easily using Docker. Make sure you have Docker and Docker Compose installed. + +``` +git clone git@github.com:JPHutchins/open-energy-view.git +cd open-energy-view +docker-compose up +``` + +On your host OS, open a Chrome or Firefox web browser and navigate to `http://localhost:5000` + +# Production + ## Environment Setup (Ubuntu 20.04) Process notes here: https://github.com/JPHutchins/open-energy-view/issues/31 @@ -73,10 +87,8 @@ cd open-energy-view sudo rabbitmqctl add_vhost myvhost sudo rabbitmqctl set_permissions -p myvhost jp ".*" ".*" ".*" ``` - ### Install frontend dependencies and build * **Install nvm** (if you don't have it) - notes: https://github.com/nvm-sh/nvm ``` curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash @@ -99,23 +111,6 @@ cd open-energy-view npm run build ``` -### Run the development server and workers -* **Start the server and workers** - * Open four terminals (example from VSCode) - - ![Four-Terminals](/docs/four-terminals.png) - * First terminal: `./run-wsgi-dev` - * Second terminal: `./run-io-worker` - * Third terminal: `./run-cpu-worker` -* **Open the development site in a browser** - * Fourth terminal: `ip a` - * Note the IP address of your WSL2 instance, in this case `172.31.30.203` - - ![ip-a](/docs/ip-a.png) - * On your host OS, open a Chrome or Firefox web browser and navigate to `http://:5000` - - ![browser-address](/docs/browser-address.png) - ### Example account setup For first time setup you must register a user to your local database. Use something easily memorable and keep in mind that you can register as many users as you need while testing. * Click "Register now!" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..59562b9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,103 @@ +version: "3.9" +services: + rabbitmq: + container_name: oev-rabbit + image: rabbitmq:3.11-alpine + ports: + - "5672:5672" + - "15672:15672" + hostname: "rabbitmq" # Important for how persistence works. + volumes: + - oev-rabbit:/var/lib/rabbitmq + rabbitmqsetup: + image: rabbitmq:3.11-alpine + restart: "no" + entrypoint: ["bash", "-c", "sleep 10 && rabbitmqctl -n rabbit@rabbitmq add_user jp admin && rabbitmqctl -n rabbit@rabbitmq set_user_tags jp administrator && rabbitmqctl -n rabbit@rabbitmq add_vhost myvhost && rabbitmqctl -n rabbit@rabbitmq set_permissions -p myvhost jp \".*\" \".*\" \".*\""] + volumes: + - oev-rabbit:/var/lib/rabbitmq + depends_on: + - rabbitmq # Make use of the cookie in the main container so we can run rabbitmqctl commands. + app: + container_name: oev + build: + context: ./ + dockerfile: ./docker/Dockerfile-app + ports: + - "5000:5000" + environment: + - FLASK_CONFIG=config.DevConfig + - FLASK_APP=open_energy_view + - FLASK_ENV=production + - SECRET_KEY=dev + - JWT_SECRET_KEY=dev + - JWT_BLACKLIST_ENABLED=True + - JWT_TOKEN_LOCATION=cookies + - JWT_ACCESS_COOKIE_PATH=/ + - JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh + - JWT_COOKIE_CSRF_PROTECT=True + - PROD_JWT_COOKIE_SECURE=True + - DEV_JWT_COOKIE_SECURE=False + - PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db + - DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db + - PGE_CLIENT_ID=client_id + - PGE_CLIENT_SECRET=client_secret + - GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID + - GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET + - OAUTHLIB_INSECURE_TRANSPORT=True + - CERT_PATH=test/cert/cert.crt + - KEY_PATH=test/cert/private.key + - API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo= + - PORT=5000 + depends_on: + - rabbitmqsetup + cpu-worker: + restart: unless-stopped + container_name: oev-cpu-worker + build: + context: ./ + dockerfile: ./docker/Dockerfile-worker + environment: + - HOSTNAME=celery.cpus@%h + - QUEUES=cpu + - CONCURRENCY=1 + - POOL=prefork + - FLASK_CONFIG=config.DevConfig + - FLASK_APP=open_energy_view + - FLASK_ENV=production + - SECRET_KEY=dev + - JWT_SECRET_KEY=dev + - JWT_BLACKLIST_ENABLED=True + - JWT_TOKEN_LOCATION=cookies + - JWT_ACCESS_COOKIE_PATH=/ + - JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh + - JWT_COOKIE_CSRF_PROTECT=True + - PROD_JWT_COOKIE_SECURE=True + - DEV_JWT_COOKIE_SECURE=False + - PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db + - DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db + - PGE_CLIENT_ID=client_id + - PGE_CLIENT_SECRET=client_secret + - GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID + - GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET + - OAUTHLIB_INSECURE_TRANSPORT=True + - CERT_PATH=test/cert/cert.crt + - KEY_PATH=test/cert/private.key + - API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo= + - PORT=5000 + depends_on: + - app + io-worker: + restart: unless-stopped + container_name: oev-io-worker + build: + context: ./ + dockerfile: ./docker/Dockerfile-worker + environment: + - HOSTNAME=celery.io@%h + - QUEUES=io + - CONCURRENCY=500 + - POOL=gevent + depends_on: + - app +volumes: + oev-rabbit: diff --git a/docker/Dockerfile-app b/docker/Dockerfile-app new file mode 100644 index 0000000..3aaa592 --- /dev/null +++ b/docker/Dockerfile-app @@ -0,0 +1,14 @@ +FROM node:10.19.0-slim AS ui +ADD open_energy_view/frontend /frontend +WORKDIR /frontend +RUN npm ci && npm run build + +FROM python:3.8-slim +ADD . /app +WORKDIR /app +RUN apt update && apt install -y build-essential +RUN pip3 install -r ./requirements.txt +RUN apt purge -y build-essential +COPY --from=ui /frontend/dist /app/open_energy_view/frontend/dist +COPY --from=ui /frontend/node_modules /app/open_energy_view/frontend/node_modules +ENTRYPOINT uwsgi --http 0.0.0.0:"$PORT" --gevent 100 --wsgi-file wsgi.py --callable app diff --git a/docker/Dockerfile-worker b/docker/Dockerfile-worker new file mode 100644 index 0000000..d9adb89 --- /dev/null +++ b/docker/Dockerfile-worker @@ -0,0 +1,2 @@ +FROM open-energy-view_app +ENTRYPOINT celery worker --app open_energy_view.celery_tasks --hostname="$HOSTNAME" --queues="$QUEUES" --loglevel=info --pool="$POOL" --concurrency="$CONCURRENCY" diff --git a/docs/browser-address.png b/docs/browser-address.png deleted file mode 100644 index ea0f8b7..0000000 Binary files a/docs/browser-address.png and /dev/null differ diff --git a/docs/ip-a.png b/docs/ip-a.png deleted file mode 100644 index 8203612..0000000 Binary files a/docs/ip-a.png and /dev/null differ diff --git a/open_energy_view/celery.py b/open_energy_view/celery.py index 3ca1f75..ce5b697 100644 --- a/open_energy_view/celery.py +++ b/open_energy_view/celery.py @@ -2,7 +2,7 @@ celery = Celery( - "tasks", backend="rpc://", broker="amqp://jp:admin@localhost:5672/myvhost", + "tasks", backend="rpc://", broker="amqp://jp:admin@rabbitmq:5672/myvhost", ) diff --git a/open_energy_view/celery_tasks.py b/open_energy_view/celery_tasks.py index a0c3910..4ef3747 100644 --- a/open_energy_view/celery_tasks.py +++ b/open_energy_view/celery_tasks.py @@ -98,7 +98,7 @@ def add(x, y): @celery.task(bind=True, name="get_jp") def get_jp(self): response = requests.get( - "http://slowwly.robertomurray.co.uk/delay/2500/url/https://www.jphutchins.com" + "https://httpstat.us/200?sleep=2500" ) return "BOOM" @@ -140,19 +140,19 @@ def fetch_task(self, published_period_start, interval_block_url, headers, cert): @celery.task(bind=True, name="fake_fetch") def fake_fetch(self): test_xml = [ - "/home/jp/open-energy-view/test/data/espi/espi_2_years.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-16.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-17.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-18.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-19.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-20.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-21.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-22.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-23.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-24.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-25.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-26.xml", - "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-27.xml", + "/app/test/data/espi/espi_2_years.xml", + "/app/test/data/espi/Single Days/2019-10-16.xml", + "/app/test/data/espi/Single Days/2019-10-17.xml", + "/app/test/data/espi/Single Days/2019-10-18.xml", + "/app/test/data/espi/Single Days/2019-10-19.xml", + "/app/test/data/espi/Single Days/2019-10-20.xml", + "/app/test/data/espi/Single Days/2019-10-21.xml", + "/app/test/data/espi/Single Days/2019-10-22.xml", + "/app/test/data/espi/Single Days/2019-10-23.xml", + "/app/test/data/espi/Single Days/2019-10-24.xml", + "/app/test/data/espi/Single Days/2019-10-25.xml", + "/app/test/data/espi/Single Days/2019-10-26.xml", + "/app/test/data/espi/Single Days/2019-10-27.xml", ] test_xml.reverse() diff --git a/open_energy_view/resources.py b/open_energy_view/resources.py index c4af402..7fbd603 100644 --- a/open_energy_view/resources.py +++ b/open_energy_view/resources.py @@ -36,7 +36,7 @@ from .celery_tasks import get_jp, insert_espi_xml_into_db, process_data from .celery import celery -IP_AND_PORT = os.environ.get("IP_AND_PORT") +PORT = os.environ.get("PORT") PGE_BULK_ID = 51070 PGE_CLIENT_ID = os.environ.get("PGE_CLIENT_ID") @@ -429,8 +429,7 @@ def get(self): class FakeOAuthStart(Resource): def get(self): - # TODO: get host IP on Dev mode - my WSL is not working on localhost... - return redirect(f"http://{IP_AND_PORT}/#/fake_oauth") + return redirect(f"http://127.0.0.1:{PORT}/#/fake_oauth") class AddFakeSourceFromFakeOAuthOLD(Resource): diff --git a/run-cpu-worker b/run-cpu-worker deleted file mode 100755 index dd457b8..0000000 --- a/run-cpu-worker +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -source venv/bin/activate -export FLASK_CONFIG=config.DevConfig -export FLASK_APP=open_energy_view -export FLASK_ENV=production -export SECRET_KEY=dev -export JWT_SECRET_KEY=dev -export JWT_BLACKLIST_ENABLED=True -export JWT_TOKEN_LOCATION=cookies -export JWT_ACCESS_COOKIE_PATH=/ -export JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh -export JWT_COOKIE_CSRF_PROTECT=True -export PROD_JWT_COOKIE_SECURE=True -export DEV_JWT_COOKIE_SECURE=False -export PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db -export DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db -export PGE_CLIENT_ID=client_id -export PGE_CLIENT_SECRET=client_secret -export GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID -export GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET -export OAUTHLIB_INSECURE_TRANSPORT=True -export CERT_PATH=test/cert/cert.crt -export KEY_PATH=test/cert/private.key -export API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo= -export IP_AND_PORT=$(hostname -I | awk '{print $1}'):5000 -celery worker --app open_energy_view.celery_tasks --hostname=celery.cpus@%h --queues=cpu --loglevel=info --concurrency=1 diff --git a/run-io-worker b/run-io-worker deleted file mode 100755 index 8dc16ed..0000000 --- a/run-io-worker +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -source venv/bin/activate - -celery worker --app open_energy_view.celery_tasks --hostname=celery.io@%h --queues=io --loglevel=info --pool=gevent --concurrency=500 diff --git a/run-wsgi-dev b/run-wsgi-dev deleted file mode 100755 index aa394ff..0000000 --- a/run-wsgi-dev +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -source venv/bin/activate -export FLASK_CONFIG=config.DevConfig -export FLASK_APP=open_energy_view -export FLASK_ENV=production -export SECRET_KEY=dev -export JWT_SECRET_KEY=dev -export JWT_BLACKLIST_ENABLED=True -export JWT_TOKEN_LOCATION=cookies -export JWT_ACCESS_COOKIE_PATH=/ -export JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh -export JWT_COOKIE_CSRF_PROTECT=True -export PROD_JWT_COOKIE_SECURE=True -export DEV_JWT_COOKIE_SECURE=False -export PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db -export DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db -export PGE_CLIENT_ID=client_id -export PGE_CLIENT_SECRET=client_secret -export GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID -export GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET -export OAUTHLIB_INSECURE_TRANSPORT=True -export CERT_PATH=test/cert/cert.crt -export KEY_PATH=test/cert/private.key -export API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo= -export IP_AND_PORT=$(hostname -I | awk '{print $1}'):5000 -./venv/bin/uwsgi --http 0.0.0.0:5000 --gevent 100 --wsgi-file wsgi.py --callable app diff --git a/test-gui b/test-gui deleted file mode 100755 index 89f365a..0000000 --- a/test-gui +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -source venv/bin/activate -export FLASK_CONFIG=config.DevConfig -export FLASK_APP=open_energy_view -export FLASK_ENV=production -export SECRET_KEY=dev -export JWT_SECRET_KEY=dev -export JWT_BLACKLIST_ENABLED=True -export JWT_TOKEN_LOCATION=cookies -export JWT_ACCESS_COOKIE_PATH=/ -export JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh -export JWT_COOKIE_CSRF_PROTECT=True -export PROD_JWT_COOKIE_SECURE=True -export DEV_JWT_COOKIE_SECURE=False -export PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db -export DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db -export PGE_CLIENT_ID=client_id -export PGE_CLIENT_SECRET=client_secret -export GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID -export GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET -export OAUTHLIB_INSECURE_TRANSPORT=True -export CERT_PATH=test/cert/cert.crt -export KEY_PATH=test/cert/private.key -export API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo= -flask run --host='0.0.0.0'