⬆️Maintenance: docker compose v2 (#46)

Author: sanderegg

.github/workflows/gateway.yml

@@ -30,6 +30,11 @@ env:
   # DOCKER_REGISTRY: must be one of docker.io, ghcr.io, registry.gitlab.com, <registry-name>.azurecr.io, gcr.io, <location>-docker.pkg.dev, <aws-account-number>.dkr.ecr.<region>.amazonaws.com, public.ecr.aws, <region>.ocir.io, quay.io, or custom (not tested)
   DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
 
+concurrency:
+  # ensure only one CI runs at a time
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   unit-test:
     timeout-minutes: 10
@@ -158,7 +163,7 @@ jobs:
       matrix:
         python: [3.9]
         os: [ubuntu-20.04]
-        docker_buildx: [v0.7.0]
+        docker_buildx: [v0.10.2]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 

Makefile

@@ -130,25 +130,53 @@ $(OSPARC_GATEWAY_CONFIG_FILE_HOST): services/osparc-gateway-server/config/defaul
 	@echo "WARNING ##### $@ does not exist, cloning $< as $@ ############"; cp $< $@)
 
 
+# REFERENCE: https://github.com/docker/compose/issues/9306
+# composeV2 defines specifications for docker compose to run
+# they are not 100% compatible with what docker stack deploy command expects
+# some parts have to be modified
+define generate_docker_compose_specs
+    docker --log-level=ERROR compose --env-file .env \
+        $(foreach file,$1,--file=$(file)) \
+        config \
+				| sed '/published:/s/"//g' \
+				| sed '/size:/s/"//g' \
+				| sed '1 { /name:.*/d ; }' \
+				| sed '1 i\version: \"3.9\"' \
+				| sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" \
+				> $@
+endef
+
+
 .stack-$(SWARM_STACK_NAME)-development.yml: .env $(docker-compose-configs)
 	# Creating config for stack with 'local/{service}:development' to $@
 	@export DOCKER_REGISTRY=local \
 	export DOCKER_IMAGE_TAG=development; \
-	docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --file services/docker-compose.devel.yml --log-level=ERROR config > $@
+	$(call generate_docker_compose_specs,\
+		services/docker-compose.yml \
+		services/docker-compose.local.yml \
+		services/docker-compose.devel.yml\
+		)
 
 .stack-$(SWARM_STACK_NAME)-production.yml: .env $(docker-compose-configs)
 	# Creating config for stack with 'local/{service}:production' to $@
 	@export DOCKER_REGISTRY=local;       \
 	export DOCKER_IMAGE_TAG=production; \
-	docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --log-level=ERROR config > $@
+	$(call generate_docker_compose_specs,\
+		services/docker-compose.yml \
+		services/docker-compose.local.yml\
+		)
 
 .stack-$(SWARM_STACK_NAME)-version.yml: .env $(docker-compose-configs)
 	# Creating config for stack with '$(DOCKER_REGISTRY)/{service}:${DOCKER_IMAGE_TAG}' to $@
-	@docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --log-level=ERROR config > $@
+	@$(call generate_docker_compose_specs,\
+		services/docker-compose.yml \
+		services/docker-compose.local.yml\
+		)
 
 .stack-$(SWARM_STACK_NAME)-ops.yml: .env $(docker-compose-configs)
 	# Creating config for ops stack to $@
-	@docker-compose --env-file .env --file services/docker-compose-ops.yml --log-level=ERROR config > $@
+	@$(call generate_docker_compose_specs,\
+		services/docker-compose-ops.yml)
 
 
 .PHONY: up-devel up-prod up-version up-latest
@@ -194,7 +222,7 @@ ifeq ($(target),)
 	@$(MAKE) .deploy-ops
 else
 	# deploys ONLY $(target) service
-	@docker-compose --file $< up --detach $(target)
+	@docker compose --file $< up --detach $(target)
 endif
 	@$(_show_endpoints)
 
@@ -214,6 +242,7 @@ down: ## Stops and removes stack
 	-@docker stack rm $(SWARM_STACK_NAME)-ops
 	# Removing generated docker compose configurations, i.e. .stack-*
 	-@rm $(wildcard .stack-*)
+	-@rm $(wildcard $(OSPARC_GATEWAY_CONFIG_FILE_HOST))
 
 leave: ## Forces to stop all services, networks, etc by the node leaving the swarm
 	-docker swarm leave -f
@@ -255,7 +284,7 @@ tag-latest: ## Tags last locally built production images as '${DOCKER_REGISTRY}/
 
 pull-version: .env ## pulls images from DOCKER_REGISTRY tagged as DOCKER_IMAGE_TAG
 	# Pulling images '${DOCKER_REGISTRY}/{service}:${DOCKER_IMAGE_TAG}'
-	@docker-compose --file services/docker-compose-deploy.yml pull
+	@docker compose --file services/docker-compose-deploy.yml pull
 
 
 .PHONY: push-version push-latest
@@ -268,7 +297,7 @@ push-latest: tag-latest
 push-version: tag-version
 	# pushing '${DOCKER_REGISTRY}/{service}:${DOCKER_IMAGE_TAG}'
 	@export BUILD_TARGET=undefined; \
-	docker-compose --file services/docker-compose-build.yml --file services/docker-compose-deploy.yml push
+	docker compose --file services/docker-compose-build.yml --file services/docker-compose-deploy.yml push
 
 ## ENVIRONMENT -------------------------------
 
@@ -277,7 +306,7 @@ push-version: tag-version
 .venv:
 	python3 -m venv $@
 	$@/bin/pip3 --quiet install --upgrade \
-		pip~=21.3 \
+		pip~=23.0 \
 		wheel \
 		setuptools
 
@@ -317,7 +346,7 @@ info: ## displays setup information
 	@echo ' node          : $(shell node --version 2> /dev/null || echo ERROR nodejs missing)'
 	@echo ' docker        : $(shell docker --version)'
 	@echo ' docker buildx : $(shell docker buildx version)'
-	@echo ' docker-compose: $(shell docker-compose --version)'
+	@echo ' docker-compose: $(shell docker compose --version)'
 
 
 define show-meta

README.md

@@ -39,7 +39,7 @@ The oSparc connects to that gateway and starts a Dask cluster made of a *itisfou
    git clone https://github.com/ITISFoundation/osparc-dask-gateway.git
    cd osparc-dask-gateway
    make up
-   # this should output the address and passwor on how to connect with the gateway
+   # this should output the address and password on how to connect with the gateway
    ```
 
 ## Testing the gateway

ci/helpers/show_system_versions.bash

@@ -32,7 +32,7 @@ if command -v docker; then
     docker buildx version
 fi
 
-echo "------------------------------ docker-compose -----------------------------------"
-if command -v docker-compose; then
-    docker-compose version
+echo "------------------------------ docker compose -----------------------------------"
+if command -v docker; then
+    docker compose version
 fi

services/docker-compose.local.yml

@@ -4,5 +4,4 @@ services:
     environment:
       - SC_BOOT_MODE=${SC_BOOT_MODE:-default}
     ports:
-      - "8000:8000"
       - "3100:3000" # ptvsd port

services/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.6' # Don't touch
+version: '3.9'
 services:
   osparc-gateway-server:
     image: ${DOCKER_REGISTRY:-itisfoundation}/osparc-gateway-server:${DOCKER_IMAGE_TAG:-latest}
@@ -24,7 +24,8 @@ services:
         constraints:
           - node.role == manager
 networks:
-  dask_net: null
+  dask_net:
+    name: ${SWARM_STACK_NAME:?swarm_stack_name_required}_dask_net
 
 volumes:
   sidecar_data: null

services/osparc-gateway-server/Dockerfile

@@ -77,7 +77,7 @@ RUN  --mount=type=cache,id=build-cache,target=/var/cache/apt,mode=0755,sharing=l
 RUN python -m venv "${VIRTUAL_ENV}"
 RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \
   pip install --upgrade \
-  pip~=21.3.1  \
+  pip~=23.0.1  \
   wheel \
   setuptools
 

services/osparc-gateway-server/requirements/_base.txt

@@ -21,13 +21,13 @@ bokeh==3.0.3
     # via -r requirements/_base.in
 cffi==1.15.1
     # via cryptography
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
     # via aiohttp
 colorlog==6.7.0
     # via dask-gateway-server
 contourpy==1.0.7
     # via bokeh
-cryptography==39.0.1
+cryptography==39.0.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   dask-gateway-server

services/osparc-gateway-server/requirements/_test.in

@@ -12,7 +12,7 @@
 
 # testing
 pytest
-pytest-aiohttp  # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76
+pytest-asyncio
 pytest-cov
 pytest-mock
 pytest-xdist

services/osparc-gateway-server/requirements/_test.txt

@@ -9,12 +9,11 @@ aiohttp==3.8.4
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   dask-gateway
-    #   pytest-aiohttp
 aiosignal==1.3.1
     # via
     #   -c requirements/_base.txt
     #   aiohttp
-astroid==2.14.2
+astroid==2.15.0
     # via pylint
 async-timeout==4.0.2
     # via
@@ -27,7 +26,7 @@ attrs==22.2.0
     #   pytest
 certifi==2022.12.7
     # via requests
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
     # via
     #   -c requirements/_base.txt
     #   aiohttp
@@ -75,14 +74,14 @@ exceptiongroup==1.1.0
     # via pytest
 execnet==1.9.0
     # via pytest-xdist
-faker==17.5.0
+faker==17.6.0
     # via -r requirements/_test.in
 frozenlist==1.3.3
     # via
     #   -c requirements/_base.txt
     #   aiohttp
     #   aiosignal
-fsspec==2023.1.0
+fsspec==2023.3.0
     # via dask
 heapdict==1.0.1
     # via zict
@@ -130,30 +129,27 @@ packaging==23.0
     #   pytest
 partd==1.3.0
     # via dask
-platformdirs==3.0.0
+platformdirs==3.1.0
     # via pylint
 pluggy==1.0.0
     # via pytest
 pprintpp==0.4.0
     # via pytest-icdiff
 psutil==5.9.4
     # via distributed
-pylint==2.16.2
+pylint==2.16.4
     # via -r requirements/_test.in
-pytest==7.2.1
+pytest==7.2.2
     # via
     #   -r requirements/_test.in
-    #   pytest-aiohttp
     #   pytest-asyncio
     #   pytest-cov
     #   pytest-icdiff
     #   pytest-mock
     #   pytest-watch
     #   pytest-xdist
-pytest-aiohttp==1.0.4
-    # via -r requirements/_test.in
 pytest-asyncio==0.20.3
-    # via pytest-aiohttp
+    # via -r requirements/_test.in
 pytest-cov==4.0.0
     # via -r requirements/_test.in
 pytest-icdiff==0.6

services/osparc-gateway-server/requirements/_tools.txt

@@ -41,7 +41,7 @@ pathspec==0.11.0
     # via black
 pip-tools==6.12.3
     # via -r requirements/../../../requirements/devenv.txt
-platformdirs==3.0.0
+platformdirs==3.1.0
     # via
     #   -c requirements/_test.txt
     #   black

tests/Makefile

@@ -19,11 +19,25 @@ install-dev install-prod install-ci: _check_venv_active ## install requirements
 
 
 .PHONY: tests
-tests: _check_venv_active ## runs all tests [DEV]
-	# running unit tests
+tests: _check_venv_active ## runs all tests [CI]
+	# running system tests
 	pytest \
-		-vv \
+		--asyncio-mode=auto \
 		--color=yes \
 		--durations=10 \
+		-vv \
+		$(CURDIR)
+
+.PHONY: test-dev
+tests-dev: _check_venv_active ## runs all tests [DEV]
+	# running system tests
+	@pytest \
 		--asyncio-mode=auto \
+		--color=yes \
+		--durations=10 \
+		--exitfirst \
+		--failed-first \
+		--keep-docker-up \
+		--pdb \
+		-vv \
 		$(CURDIR)

tests/requirements/_test.in

@@ -3,7 +3,7 @@
 
 # testing
 pytest
-pytest-aiohttp
+pytest-asyncio
 pytest-cov
 pytest-icdiff
 pytest-instafail