Ease development with Docker

README.md

@@ -18,6 +18,20 @@ If you are a PG&E customer you can link your account now!  If you are not a PG&E
 
 # Development
 
+## Environment Setup (Docker)
+
+You can get an environment setup easily using Docker. Make sure you have Docker and Docker Compose installed.
+
+```
+git clone [email protected]:JPHutchins/open-energy-view.git
+cd open-energy-view
+docker-compose up
+```
+
+On your host OS, open a Chrome or Firefox web browser and navigate to `http://localhost:5000`
+
+# Production
+
 ## Environment Setup (Ubuntu 20.04)
 
 Process notes here: https://github.com/JPHutchins/open-energy-view/issues/31
@@ -73,10 +87,8 @@ cd open-energy-view
     sudo rabbitmqctl add_vhost myvhost
     sudo rabbitmqctl set_permissions -p myvhost jp ".*" ".*" ".*"
     ```
-
 ### Install frontend dependencies and build
 * **Install nvm** (if you don't have it)
-
   notes: https://github.com/nvm-sh/nvm
   ```
   curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash
@@ -99,23 +111,6 @@ cd open-energy-view
   npm run build
   ```
 
-### Run the development server and workers
-* **Start the server and workers**
-  * Open four terminals (example from VSCode)
-
-  ![Four-Terminals](/docs/four-terminals.png)
-  * First terminal: `./run-wsgi-dev`
-  * Second terminal: `./run-io-worker`
-  * Third terminal: `./run-cpu-worker`
-* **Open the development site in a browser**
-  * Fourth terminal: `ip a`
-  * Note the IP address of your WSL2 instance, in this case `172.31.30.203`
-
-  ![ip-a](/docs/ip-a.png)
-  * On your host OS, open a Chrome or Firefox web browser and navigate to `http://<YOUR_WSL2_IP>:5000`
-
-  ![browser-address](/docs/browser-address.png)
-
 ### Example account setup
 For first time setup you must register a user to your local database. Use something easily memorable and keep in mind that you can register as many users as you need while testing.
 * Click "Register now!"

docker-compose.yml

@@ -0,0 +1,103 @@
+version: "3.9"
+services:
+  rabbitmq:
+    container_name: oev-rabbit
+    image: rabbitmq:3.11-alpine
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    hostname: "rabbitmq" # Important for how persistence works.
+    volumes:
+      - oev-rabbit:/var/lib/rabbitmq
+  rabbitmqsetup:
+    image: rabbitmq:3.11-alpine
+    restart: "no"
+    entrypoint: ["bash", "-c", "sleep 10 && rabbitmqctl -n rabbit@rabbitmq add_user jp admin && rabbitmqctl -n rabbit@rabbitmq set_user_tags jp administrator && rabbitmqctl -n rabbit@rabbitmq add_vhost myvhost && rabbitmqctl -n rabbit@rabbitmq set_permissions -p myvhost jp \".*\" \".*\" \".*\""]
+    volumes:
+      - oev-rabbit:/var/lib/rabbitmq
+    depends_on:
+      - rabbitmq # Make use of the cookie in the main container so we can run rabbitmqctl commands.
+  app:
+    container_name: oev
+    build:
+      context: ./
+      dockerfile: ./docker/Dockerfile-app
+    ports:
+      - "5000:5000"
+    environment:
+      - FLASK_CONFIG=config.DevConfig
+      - FLASK_APP=open_energy_view
+      - FLASK_ENV=production
+      - SECRET_KEY=dev
+      - JWT_SECRET_KEY=dev
+      - JWT_BLACKLIST_ENABLED=True
+      - JWT_TOKEN_LOCATION=cookies
+      - JWT_ACCESS_COOKIE_PATH=/
+      - JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh
+      - JWT_COOKIE_CSRF_PROTECT=True
+      - PROD_JWT_COOKIE_SECURE=True
+      - DEV_JWT_COOKIE_SECURE=False
+      - PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db
+      - DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db
+      - PGE_CLIENT_ID=client_id
+      - PGE_CLIENT_SECRET=client_secret
+      - GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID
+      - GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET
+      - OAUTHLIB_INSECURE_TRANSPORT=True
+      - CERT_PATH=test/cert/cert.crt
+      - KEY_PATH=test/cert/private.key
+      - API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo=
+      - PORT=5000
+    depends_on:
+      - rabbitmqsetup
+  cpu-worker:
+    restart: unless-stopped
+    container_name: oev-cpu-worker
+    build:
+      context: ./
+      dockerfile: ./docker/Dockerfile-worker
+    environment:
+      - HOSTNAME=celery.cpus@%h
+      - QUEUES=cpu
+      - CONCURRENCY=1
+      - POOL=prefork
+      - FLASK_CONFIG=config.DevConfig
+      - FLASK_APP=open_energy_view
+      - FLASK_ENV=production
+      - SECRET_KEY=dev
+      - JWT_SECRET_KEY=dev
+      - JWT_BLACKLIST_ENABLED=True
+      - JWT_TOKEN_LOCATION=cookies
+      - JWT_ACCESS_COOKIE_PATH=/
+      - JWT_REFRESH_COOKIE_PATH=/api/web/token/refresh
+      - JWT_COOKIE_CSRF_PROTECT=True
+      - PROD_JWT_COOKIE_SECURE=True
+      - DEV_JWT_COOKIE_SECURE=False
+      - PROD_DATABASE_URI=sqlite:///../test/data/energy_history_test.db
+      - DEV_DATABASE_URI=sqlite:///../test/data/energy_history_test.db
+      - PGE_CLIENT_ID=client_id
+      - PGE_CLIENT_SECRET=client_secret
+      - GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID
+      - GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET
+      - OAUTHLIB_INSECURE_TRANSPORT=True
+      - CERT_PATH=test/cert/cert.crt
+      - KEY_PATH=test/cert/private.key
+      - API_RESPONSE_KEY=xS5MqJ6N9CyH-hvqAGrmBVAxFMOyauMpdrdqCZa1eqo=
+      - PORT=5000
+    depends_on:
+      - app
+  io-worker:
+    restart: unless-stopped
+    container_name: oev-io-worker
+    build:
+      context: ./
+      dockerfile: ./docker/Dockerfile-worker
+    environment:
+      - HOSTNAME=celery.io@%h
+      - QUEUES=io
+      - CONCURRENCY=500
+      - POOL=gevent
+    depends_on:
+      - app
+volumes:
+  oev-rabbit:

docker/Dockerfile-app

@@ -0,0 +1,14 @@
+FROM node:10.19.0-slim AS ui
+ADD open_energy_view/frontend /frontend
+WORKDIR /frontend
+RUN npm ci && npm run build
+
+FROM python:3.8-slim
+ADD . /app
+WORKDIR /app
+RUN apt update && apt install -y build-essential
+RUN pip3 install -r ./requirements.txt
+RUN apt purge -y build-essential
+COPY --from=ui /frontend/dist /app/open_energy_view/frontend/dist
+COPY --from=ui /frontend/node_modules /app/open_energy_view/frontend/node_modules
+ENTRYPOINT uwsgi --http 0.0.0.0:"$PORT" --gevent 100 --wsgi-file wsgi.py --callable app

docker/Dockerfile-worker

@@ -0,0 +1,2 @@
+FROM open-energy-view_app
+ENTRYPOINT celery worker --app open_energy_view.celery_tasks --hostname="$HOSTNAME" --queues="$QUEUES" --loglevel=info --pool="$POOL" --concurrency="$CONCURRENCY"

open_energy_view/celery.py

@@ -2,7 +2,7 @@
 
 
 celery = Celery(
-    "tasks", backend="rpc://", broker="amqp://jp:admin@localhost:5672/myvhost",
+    "tasks", backend="rpc://", broker="amqp://jp:admin@rabbitmq:5672/myvhost",
 )
 
 

open_energy_view/celery_tasks.py

@@ -98,7 +98,7 @@ def add(x, y):
 @celery.task(bind=True, name="get_jp")
 def get_jp(self):
     response = requests.get(
-        "http://slowwly.robertomurray.co.uk/delay/2500/url/https://www.jphutchins.com"
+        "https://httpstat.us/200?sleep=2500"
     )
     return "BOOM"
 
@@ -140,19 +140,19 @@ def fetch_task(self, published_period_start, interval_block_url, headers, cert):
 @celery.task(bind=True, name="fake_fetch")
 def fake_fetch(self):
     test_xml = [
-        "/home/jp/open-energy-view/test/data/espi/espi_2_years.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-16.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-17.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-18.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-19.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-20.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-21.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-22.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-23.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-24.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-25.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-26.xml",
-        "/home/jp/open-energy-view/test/data/espi/Single Days/2019-10-27.xml",
+        "/app/test/data/espi/espi_2_years.xml",
+        "/app/test/data/espi/Single Days/2019-10-16.xml",
+        "/app/test/data/espi/Single Days/2019-10-17.xml",
+        "/app/test/data/espi/Single Days/2019-10-18.xml",
+        "/app/test/data/espi/Single Days/2019-10-19.xml",
+        "/app/test/data/espi/Single Days/2019-10-20.xml",
+        "/app/test/data/espi/Single Days/2019-10-21.xml",
+        "/app/test/data/espi/Single Days/2019-10-22.xml",
+        "/app/test/data/espi/Single Days/2019-10-23.xml",
+        "/app/test/data/espi/Single Days/2019-10-24.xml",
+        "/app/test/data/espi/Single Days/2019-10-25.xml",
+        "/app/test/data/espi/Single Days/2019-10-26.xml",
+        "/app/test/data/espi/Single Days/2019-10-27.xml",
     ]
     test_xml.reverse()
 

open_energy_view/resources.py

@@ -36,7 +36,7 @@
 from .celery_tasks import get_jp, insert_espi_xml_into_db, process_data
 from .celery import celery
 
-IP_AND_PORT = os.environ.get("IP_AND_PORT")
+PORT = os.environ.get("PORT")
 
 PGE_BULK_ID = 51070
 PGE_CLIENT_ID = os.environ.get("PGE_CLIENT_ID")
@@ -429,8 +429,7 @@ def get(self):
 
 class FakeOAuthStart(Resource):
     def get(self):
-        # TODO: get host IP on Dev mode - my WSL is not working on localhost...
-        return redirect(f"http://{IP_AND_PORT}/#/fake_oauth")
+        return redirect(f"http://127.0.0.1:{PORT}/#/fake_oauth")
 
 
 class AddFakeSourceFromFakeOAuthOLD(Resource):