feat(docker compose): celery & flower

.env.example

@@ -1,11 +1,19 @@
 POSTGRES_DIALECT_DRIVER=postgresql+psycopg
 POSTGRES_USER=admin
 POSTGRES_PASSWORD=admin
-POSTGRES_HOST=localhost
+POSTGRES_HOST=postgres
 POSTGRES_DB=desbordante
 POSTGRES_PORT=5432
 
 RABBITMQ_DEFAULT_USER=guest
 RABBITMQ_DEFAULT_PASSWORD=guest
-RABBITMQ_HOST=localhost
+RABBITMQ_HOST=rabbitmq
 RABBITMQ_PORT=5672
+RABBITMQ_HTTP_PORT=15672
+
+BACKEND_PORT=8000
+UPLOADED_FILES_DIR_PATH=/volumes/uploads
+
+FLOWER_USER=admin
+FLOWER_PASSWORD=admin
+FLOWER_PORT=5555

.gitignore

@@ -113,3 +113,7 @@ coverage-report-html/
 
 .mypy_cache/
 .ruff_cache/
+
+# Volumes
+volumes/*
+!volumes/.gitkeep

Dockerfile

@@ -59,6 +59,18 @@ ENV PATH="$VIRTUAL_ENVIRONMENT_PATH/bin:$PATH"
 RUN groupadd -g 1001 python_application && \
     useradd -r -u 1001 -g python_application python_application
 
+COPY ./scripts/start.sh /start
+RUN sed -i 's/\r$//g' /start
+RUN chmod +x /start
+
+COPY ./scripts/celery.sh /celery
+RUN sed -i 's/\r$//g' /celery
+RUN chmod +x /celery
+
+COPY ./scripts/flower.sh /flower
+RUN sed -i 's/\r$//g' /flower
+RUN chmod +x /flower
+
 # Set the WORKDIR to the application root.
 # https://www.uvicorn.org/settings/#development
 # https://docs.docker.com/engine/reference/builder/#workdir
@@ -78,7 +90,8 @@ EXPOSE ${APPLICATION_SERVER_PORT}
 USER 1001
 
 # Run the uvicorn application server.
-CMD exec uvicorn --workers 1 --host 0.0.0.0 --port $APPLICATION_SERVER_PORT internal:app
+CMD /start
+
 
 FROM server-setup-build-stage as install-dependencies-build-stage
 # install [tool.poetry.dependencies]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: env install-deps up open-db pg-revision pg-migrate pg-downgrade celery-worker app init lint test check-types
+.PHONY: env volumes install-deps up open-db pg-revision pg-migrate pg-downgrade celery-worker app init lint test check-types
 
 ifeq ($(shell test -e '.env' && echo -n yes), yes)
 	include .env
@@ -12,6 +12,13 @@ env:
 	@echo >> .env
 	@echo "SECRET_KEY=$$(openssl rand -hex 32)" >> .env
 
+## Create folders for volumes
+volumes:
+	@for volume in postgres rabbitmq uploads; do \
+    	mkdir -p ./volumes/$$volume; \
+    	chmod 777 ./volumes/$$volume; \
+    done
+
 ## Install dependencies
 install-deps:
 	poetry install
@@ -40,15 +47,15 @@ pg-downgrade:
 
 ## Run celery worker in watch mode
 celery-worker:
-	watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A internal.infrastructure.background_task.celery worker --loglevel=info --concurrency=1
+	poetry run watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A internal.infrastructure.background_task.celery worker --loglevel=info --concurrency=1
 
 ## Run application server in watch mode
 app:
 	poetry run uvicorn --port 8000 internal:app --reload
 
 ## Initiate repository
 init:
-	make env install-deps
+	make env volumes install-deps
 
 ## Run all formatters and linters in project
 lint:

dev-docker-compose.yaml

@@ -9,6 +9,8 @@ services:
       - .env
     ports:
       - '${POSTGRES_PORT}:5432'
+    volumes:
+      - ./volumes/postgres:/var/lib/postgresql/data
 
   rabbitmq:
     container_name: desbordante-rabbitmq
@@ -17,3 +19,48 @@ services:
       - .env
     ports:
       - "${RABBITMQ_PORT}:5672"
+      - "${RABBITMQ_HTTP_PORT}:15672"
+    volumes:
+      - ./volumes/rabbitmq:/var/lib/rabbitmq
+
+#  backend:
+#    build:
+#      context: .
+#    volumes:
+#      - ./volumes/uploads:${UPLOADED_FILES_DIR_PATH}
+#    env_file:
+#      - .env
+#    depends_on:
+#      - postgres
+#      - rabbitmq
+#    restart: always
+#    ports:
+#      - "${BACKEND_PORT}:8000"
+
+  celery:
+    build:
+      context: .
+    command: /celery
+    volumes:
+      - ./volumes/uploads:${UPLOADED_FILES_DIR_PATH}
+    env_file:
+      - .env
+    depends_on:
+      - rabbitmq
+    restart: always
+
+
+  flower:
+    build:
+      context: .
+    command: /flower
+    volumes:
+      - ./volumes/uploads:${UPLOADED_FILES_DIR_PATH}
+    env_file:
+      - .env
+    depends_on:
+      - rabbitmq
+      - celery
+    restart: always
+    ports:
+      - "${FLOWER_PORT}:5555"

pyproject.toml

@@ -25,6 +25,7 @@ pandas-stubs = "^2.2.0.240218"
 python-multipart = "^0.0.9"
 aiofiles = "^23.2.1"
 cfgv = "^3.4.0"
+flower = "^2.0.1"
 
 
 [tool.poetry.group.dev.dependencies]

scripts/celery.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+celery -A internal.infrastructure.background_task.celery worker --loglevel=info

scripts/flower.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+worker_ready() {
+    celery -A internal.infrastructure.background_task.celery inspect ping
+}
+
+until worker_ready; do
+  >&2 echo 'Celery workers not available'
+  sleep 1
+done
+>&2 echo 'Celery workers is available'
+
+celery \
+    --app=internal.infrastructure.background_task.celery \
+    flower \
+    --port=5555
+    --basic-auth=${FLOWER_USER}:${FLOWER_PASSWORD}

scripts/start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+alembic -c internal/infrastructure/data_storage/relational/postgres/migrations/alembic.ini upgrade head
+uvicorn --workers 1 --host 0.0.0.0 --port $APPLICATION_SERVER_PORT internal:app