diff --git a/.github/workflows/stack-integration_tests.yml b/.github/workflows/stack-integration_tests.yml index 358387b6775..47beb8fae2a 100644 --- a/.github/workflows/stack-integration_tests.yml +++ b/.github/workflows/stack-integration_tests.yml @@ -81,6 +81,81 @@ jobs: run: | tox -e stack.test.integration + stack-integration-tests-tls: + strategy: + max-parallel: 3 + matrix: + os: [ubuntu-latest] + python-version: [3.9] + + runs-on: ${{matrix.os}} + + steps: + - uses: actions/checkout@v2 + + - name: Check for file changes + uses: dorny/paths-filter@v2 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + if: steps.changes.outputs.stack == 'true' + with: + python-version: ${{ matrix.python-version }} + + - name: Get pip cache dir + if: steps.changes.outputs.stack == 'true' + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@v2 + if: steps.changes.outputs.stack == 'true' + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }} + + - name: Upgrade pip + if: steps.changes.outputs.stack == 'true' + run: | + pip install --upgrade --user pip + + - name: Install tox + if: steps.changes.outputs.stack == 'true' + run: | + pip install tox --upgrade + + - name: Install Docker Compose + if: runner.os == 'Linux' + shell: bash + run: | + mkdir -p ~/.docker/cli-plugins + DOCKER_COMPOSE_VERSION=v2.1.1 + curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose + chmod +x ~/.docker/cli-plugins/docker-compose + + - name: Install mkcert + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt install libnss3-tools -y + MKCERT_VERSION=v1.4.3 + curl -sSL https://github.com/FiloSottile/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-linux-amd64 -o /usr/local/bin/mkcert + chmod +x /usr/local/bin/mkcert + which mkcert + + - name: Run integration tests + if: steps.changes.outputs.stack == 'true' + timeout-minutes: 30 + run: | + tox -e stack.test.integration.tls + stack-integration-tests-windows: strategy: max-parallel: 3 @@ -155,7 +230,7 @@ jobs: pip install -e packages/hagrid set HAGRID_ART=false hagrid launch test_network_1 network to docker:9081 --tail=false --headless=true - hagrid launch test_domain_1 domain to docker:9082 --tail=false --build=false --headless=true + hagrid launch test_domain_1 domain to docker:9082 --tail=false --headless=true hagrid launch test_domain_2 domain to docker:9083 --tail=false --build=false --headless=true bash -c "(docker logs test_domain_1-backend_stream-1 -f &) | grep -q 'Application startup complete' || true" bash -c "(docker logs test_domain_2-backend_stream-1 -f &) | grep -q 'Application startup complete' || true" diff --git a/packages/grid/.env b/packages/grid/.env index 3e4f73ad1fd..ef8619984de 100644 --- a/packages/grid/.env +++ b/packages/grid/.env @@ -1,10 +1,12 @@ #!/bin/bash DOMAIN=localhost -DOMAIN_NAME=grid.openmined.org +DOMAIN_NAME=default_node_name NODE_TYPE=domain -DOMAIN_PORT=80 +HTTP_PORT=80 +HTTPS_PORT=443 HEADSCALE_PORT=8080 NETWORK_NAME=omnet +IGNORE_TLS_ERRORS=False STACK_NAME=grid-openmined-org TRAEFIK_PUBLIC_NETWORK=traefik-public @@ -15,6 +17,7 @@ DOCKER_IMAGE_BACKEND=openmined/grid-backend DOCKER_IMAGE_FRONTEND=openmined/grid-frontend DOCKER_IMAGE_HEADSCALE=openmined/grid-vpn-headscale DOCKER_IMAGE_TAILSCALE=openmined/grid-vpn-tailscale +DOCKER_IMAGE_TRAEFIK=traefik:v2.5 VERSION=latest VERSION_HASH=unknown STACK_API_KEY=hex_key_value @@ -31,7 +34,7 @@ SMTP_HOST= SMTP_USER= SMTP_PASSWORD= EMAILS_FROM_EMAIL=info@openmined.org -SERVER_HOST="http://${DOMAIN}" +SERVER_HOST="https://${DOMAIN}" USERS_OPEN_REGISTRATION=False diff --git a/packages/grid/.gitignore b/packages/grid/.gitignore index 69762a83270..67fe27c9d01 100644 --- a/packages/grid/.gitignore +++ b/packages/grid/.gitignore @@ -9,6 +9,7 @@ packer/output/* packer/packer_cache/* packer/base-manifest.json packer/azure_vars.json +tls/ # devspace -.devspace/ \ No newline at end of file +.devspace/ diff --git a/packages/grid/.gitlab-ci.yml b/packages/grid/.gitlab-ci.yml deleted file mode 100644 index 834f4ad6ec8..00000000000 --- a/packages/grid/.gitlab-ci.yml +++ /dev/null @@ -1,74 +0,0 @@ -image: tiangolo/docker-with-compose - -before_script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - pip install docker-auto-labels - -stages: - - test - - build - - deploy - -tests: - stage: test - script: - - sh ./scripts/test.sh - tags: - - build - - test - -build-stag: - stage: build - script: - - TAG=stag FRONTEND_ENV=staging sh ./scripts/build-push.sh - only: - - master - tags: - - build - - test - -build-prod: - stage: build - script: - - TAG=prod FRONTEND_ENV=production sh ./scripts/build-push.sh - only: - - production - tags: - - build - - test - -deploy-stag: - stage: deploy - script: - - > - DOMAIN=stag.grid.openmined.org - TRAEFIK_TAG=stag.grid.openmined.org - STACK_NAME=stag-grid-openmined-org - TAG=stag - sh ./scripts/deploy.sh - environment: - name: staging - url: https://stag.grid.openmined.org - only: - - master - tags: - - swarm - - stag - -deploy-prod: - stage: deploy - script: - - > - DOMAIN=grid.openmined.org - TRAEFIK_TAG=grid.openmined.org - STACK_NAME=grid-openmined-org - TAG=prod - sh ./scripts/deploy.sh - environment: - name: production - url: https://grid.openmined.org - only: - - production - tags: - - swarm - - prod diff --git a/packages/grid/backend/grid/api/meta/ping.py b/packages/grid/backend/grid/api/meta/ping.py index 92f64eafa0d..e99c6a28546 100644 --- a/packages/grid/backend/grid/api/meta/ping.py +++ b/packages/grid/backend/grid/api/meta/ping.py @@ -11,6 +11,7 @@ # syft absolute from syft.core.node.common.node_service.ping.ping_messages import PingMessageWithReply +from syft.grid import GridURL # grid absolute from grid.api.dependencies.current_user import get_current_user @@ -27,7 +28,7 @@ def remote_ping( # Build Syft Message msg = ( - PingMessageWithReply(kwargs={"host_or_ip": host_or_ip}) + PingMessageWithReply(kwargs={"grid_url": GridURL.from_url(host_or_ip)}) .to(address=node.address, reply_to=node.address) .sign(signing_key=user_key) ) diff --git a/packages/grid/backend/grid/api/vpn/vpn.py b/packages/grid/backend/grid/api/vpn/vpn.py index 9b536f17a4a..0033f87a7ee 100644 --- a/packages/grid/backend/grid/api/vpn/vpn.py +++ b/packages/grid/backend/grid/api/vpn/vpn.py @@ -21,6 +21,7 @@ VPNStatusMessageWithReply, ) from syft.core.node.common.node_service.vpn.vpn_messages import VPNJoinMessageWithReply +from syft.grid import GridURL from syft.lib.python.util import upcast # grid absolute @@ -44,7 +45,7 @@ def connect( msg = ( VPNConnectMessageWithReply( kwargs={ - "host_or_ip": host_or_ip, + "grid_url": GridURL.from_url(host_or_ip), "vpn_auth_key": vpn_auth_key, } ) @@ -72,7 +73,7 @@ def join( ) -> Dict[str, Any]: user_key = SigningKey(current_user.private_key.encode(), encoder=HexEncoder) msg = ( - VPNJoinMessageWithReply(kwargs={"host_or_ip": host_or_ip}) + VPNJoinMessageWithReply(kwargs={"grid_url": GridURL.from_url(host_or_ip)}) .to(address=node.address, reply_to=node.address) .sign(signing_key=user_key) ) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 5f758b331a0..13d3917b6ce 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -20,7 +20,6 @@ class Settings(BaseSettings): SECRET_KEY: str = secrets.token_urlsafe(32) # 60 minutes * 24 hours * 8 days = 8 days ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8 - SERVER_NAME: str = "unconfigured" SERVER_HOST: str = "https://localhost" # BACKEND_CORS_ORIGINS is a JSON-formatted list of origins # e.g: '["http://localhost", "http://localhost:4200", "http://localhost:3000", \ @@ -93,7 +92,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: FIRST_SUPERUSER_PASSWORD: str = "changethis" USERS_OPEN_REGISTRATION: bool = False - DOMAIN_NAME: str = "grid_domain" + DOMAIN_NAME: str = "default_node_name" STREAM_QUEUE: bool = False NODE_TYPE: str = "Domain" diff --git a/packages/grid/backend/grid/tests/conftest.py b/packages/grid/backend/grid/tests/conftest.py index a7067c09f44..3e0e390b672 100644 --- a/packages/grid/backend/grid/tests/conftest.py +++ b/packages/grid/backend/grid/tests/conftest.py @@ -1,6 +1,5 @@ # stdlib import logging -import os from typing import Generator # third party @@ -56,17 +55,3 @@ def emit(self, record: logging.LogRecord) -> None: sink_handler_id = logger.add(PropagateHandler(), format=log_handler.format_record) yield caplog logger.remove(sink_handler_id) - - -# patch windows to use uft-8 output -if os.name == "nt": - try: - print("Patching Windows Default Locale to use UTF-8") - # third party - import _locale - - _locale._gdl_bak = _locale._getdefaultlocale - _locale._getdefaultlocale = lambda *args: (_locale._gdl_bak()[0], "utf8") - print("Finished Patching Windows Default Locale to use UTF-8") - except Exception as e: - print(f"Failed to patch Windows Default Locale. {e}") diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 80ec00ff788..8e4d7e260e9 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -148,8 +148,6 @@ deployments: successThreshold: 1 failureThreshold: 3 env: - - name: SYFT_USE_UVLOOP - value: "0" - name: DOMAIN_NAME value: ${DOMAIN_NAME} - name: POSTGRES_SERVER @@ -164,14 +162,14 @@ deployments: value: ${VERSION} - name: VERSION_HASH value: ${VERSION_HASH} - - name: SERVER_NAME - value: ${DOMAIN} - name: SERVER_HOST value: ${SERVER_HOST} - name: LOG_LEVEL value: debug - name: NODE_TYPE value: ${NODE_TYPE} + - name: STACK_API_KEY + value: ${STACK_API_KEY} service: name: ${SERVICE_NAME_BACKEND} ports: @@ -209,8 +207,6 @@ deployments: successThreshold: 1 failureThreshold: 3 env: - - name: SYFT_USE_UVLOOP - value: "0" - name: DOMAIN_NAME value: ${DOMAIN_NAME} - name: POSTGRES_SERVER @@ -225,8 +221,6 @@ deployments: value: ${VERSION} - name: VERSION_HASH value: ${VERSION_HASH} - - name: SERVER_NAME - value: ${DOMAIN} - name: SERVER_HOST value: ${SERVER_HOST} - name: LOG_LEVEL @@ -235,6 +229,8 @@ deployments: value: "1" - name: NODE_TYPE value: ${NODE_TYPE} + - name: STACK_API_KEY + value: ${STACK_API_KEY} service: name: ${SERVICE_NAME_BACKEND_STREAM} ports: @@ -256,8 +252,6 @@ deployments: "/worker-start.sh", ] env: - - name: SYFT_USE_UVLOOP - value: "0" - name: DOMAIN_NAME value: ${DOMAIN_NAME} - name: POSTGRES_SERVER @@ -272,8 +266,6 @@ deployments: value: ${VERSION} - name: VERSION_HASH value: ${VERSION_HASH} - - name: SERVER_NAME - value: ${DOMAIN} - name: SERVER_HOST value: ${SERVER_HOST} - name: CELERY_WORKER @@ -284,6 +276,8 @@ deployments: value: ${NODE_TYPE} - name: C_FORCE_ROOT value: "1" + - name: STACK_API_KEY + value: ${STACK_API_KEY} - name: frontend helm: componentChart: true @@ -295,7 +289,7 @@ deployments: value: ${VERSION} - name: VERSION_HASH value: ${VERSION_HASH} - - name: TYPE + - name: NODE_TYPE value: ${NODE_TYPE} service: name: ${SERVICE_NAME_FRONTEND} @@ -310,6 +304,8 @@ deployments: env: - name: NETWORK_NAME value: ${NETWORK_NAME} + - name: STACK_API_KEY + value: ${STACK_API_KEY} volumeMounts: - containerPath: /headscale/data volume: @@ -339,6 +335,8 @@ deployments: env: - name: HOSTNAME value: ${DOMAIN_NAME} + - name: STACK_API_KEY + value: ${STACK_API_KEY} volumeMounts: - containerPath: /var/lib/tailscale volume: diff --git a/packages/grid/docker-compose.override.yml b/packages/grid/docker-compose.override.yml deleted file mode 100644 index 83564b48a71..00000000000 --- a/packages/grid/docker-compose.override.yml +++ /dev/null @@ -1,209 +0,0 @@ -version: "3.8" -services: - proxy: - # ports: - # - "${DOMAIN_PORT?80}:80" - # - "8080" - command: - # Enable Docker in Traefik, so that it reads labels from Docker services - - --providers.docker - # Add a constraint to only use services with the label for this stack - # from the env var TRAEFIK_TAG - - --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`) - # Do not expose all Docker services, only the ones explicitly exposed - - --providers.docker.exposedbydefault=false - # Disable Docker Swarm mode for local development - # - --providers.docker.swarmmode - # Enable the access log, with HTTP requests - - --accesslog - # Enable the Traefik log, for configurations and errors - - --log - # Enable the Dashboard and API - - --api - # Enable the Dashboard and API in insecure mode for local development - - --api.insecure=true - - --api.dashboard=true - labels: - - traefik.enable=true - - traefik.http.routers.${STACK_NAME?Variable not set}-traefik-public-http.rule=Host(`${DOMAIN?Variable not set}`) - - traefik.http.services.${STACK_NAME?Variable not set}-traefik-public.loadbalancer.server.port=80 - network_mode: service:tailscale - networks: - - "${TRAEFIK_PUBLIC_NETWORK?Variable not set}" - - default - - db: - ports: - - "5432" - - queue: - image: rabbitmq:3-management - ports: - - "5672" - - "15672" - volumes: - - ./rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - - backend: - volumes: - - ./backend/grid:/app/grid - - ./backend/alembic:/app/alembic - - ../syft:/app/syft - - ../../notebooks:/notebooks - - ./data/package-cache:/root/.cache - - environment: - - SERVER_HOST=http://${DOMAIN?Variable not set} - - SYFT_USE_UVLOOP=0 - - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} - - NODE_TYPE=${NODE_TYPE?Variable not set} - - PORT=8001 - - STACK_API_KEY=$STACK_API_KEY - - build: - context: ../ - dockerfile: ./grid/backend/backend.dockerfile - target: "backend" - # command: bash -c "while true; do sleep 1; done" # Infinite loop to keep container live doing nothing - command: /start-reload.sh - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`) # WARNING: this wont match /api/v1/syft/stream because of length - - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=8001 - network_mode: service:proxy - - backend_stream: - depends_on: - - db - - backend - volumes: - - ./backend/grid:/app/grid - - ./backend/alembic:/app/alembic - - ../syft:/app/syft - - ./data/package-cache:/root/.cache - environment: - - SERVER_HOST=http://${DOMAIN?Variable not set} - - SYFT_USE_UVLOOP=0 - - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} - - NODE_TYPE=${NODE_TYPE?Variable not set} - - STREAM_QUEUE=1 - - PORT=8011 - - STACK_API_KEY=$STACK_API_KEY - - build: - context: ../ - dockerfile: ./grid/backend/backend.dockerfile - target: "backend" - # command: bash -c "while true; do sleep 1; done" # Infinite loop to keep container live doing nothing - command: "waitforit -address=http://localhost:8001/api/v1/syft/metadata -status=200 -timeout=600 -- /start-reload.sh" - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-backend-stream-http.rule=PathPrefix(`/api`) && PathPrefix(`/api/v1/syft/stream`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`) # WARNING: this only matches /api/v1/syft/stream because of length - - traefik.http.services.${STACK_NAME?Variable not set}-backend-stream.loadbalancer.server.port=8011 - network_mode: service:proxy - - celeryworker: - depends_on: - - backend - volumes: - - ./backend/grid:/app/grid #- ./backend/app:/app - - ./backend/alembic:/app/alembic - - ../syft:/app/syft #- ../syft:/app/syft - - ../../notebooks:/notebooks - - ./data/package-cache:/root/.cache - environment: - - RUN=celery -A grid.worker worker -l info -Q main-queue --pool=gevent -c 500 - - SERVER_HOST=http://${DOMAIN?Variable not set} - - SYFT_USE_UVLOOP=0 - - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} - - C_FORCE_ROOT=1 - - STACK_API_KEY=$STACK_API_KEY - - build: - context: ../ - dockerfile: ./grid/backend/backend.dockerfile - target: "backend" - command: "waitforit -address=http://localhost:8001/api/v1/syft/metadata -status=200 -timeout=600 -- /worker-start.sh" - network_mode: service:proxy - - frontend: - profiles: - - frontend - build: - context: ./frontend - dockerfile: frontend.dockerfile - args: - FRONTEND_ENV: ${FRONTEND_ENV-development} - TYPE: ${NODE_TYPE?Variable not set} - volumes: - - ./frontend/src:/app/src - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`) - - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 - - tailscale: - hostname: ${DOMAIN_NAME?Variable not set} - image: "${DOCKER_IMAGE_TAILSCALE?Variable not set}:${VERSION-latest}" - build: - context: ./vpn - dockerfile: tailscale.dockerfile - environment: - - HOSTNAME=${DOMAIN_NAME?Variable not set} - - STACK_API_KEY=$STACK_API_KEY - - volumes: - - tailscale-data:/var/lib/tailscale - - "/dev/net/tun:/dev/net/tun" # Required for tailscale to work - # - ./grid/vpn:/tailscale - cap_add: # Required for tailscale to work - - net_admin - - sys_module - ports: - - "${DOMAIN_PORT?80}:80" - - "41641/udp" - - "4000" - depends_on: - - "docker-host" - - headscale: - profiles: - - network - hostname: headscale - image: "${DOCKER_IMAGE_HEADSCALE?Variable not set}:${VERSION-latest}" - build: - context: ./vpn - dockerfile: headscale.dockerfile - volumes: - - headscale-data:/headscale/data - # - ./grid/vpn:/headscale - environment: - - NETWORK_NAME=omnet - - STACK_API_KEY=$STACK_API_KEY - ports: - - "4000" - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-vpn-http.rule=PathPrefix(`/vpn`) - - "traefik.http.routers.${STACK_NAME?Variable not set}-vpn-http.middlewares=${STACK_NAME?Variable not set}-vpn-http-stripprefix" - - "traefik.http.middlewares.${STACK_NAME?Variable not set}-vpn-http-stripprefix.stripprefix.prefixes=/vpn" - - traefik.http.services.${STACK_NAME?Variable not set}-vpn.loadbalancer.server.port=8080 - - docker-host: - image: qoomon/docker-host - cap_add: - - net_admin - - net_raw - -networks: - traefik-public: - # For local dev, don't expect an external Traefik network - external: false - -volumes: - tailscale-data: - headscale-data: diff --git a/packages/grid/docker-compose.test.yml b/packages/grid/docker-compose.test.yml new file mode 100644 index 00000000000..d6d82a550ca --- /dev/null +++ b/packages/grid/docker-compose.test.yml @@ -0,0 +1,5 @@ +version: "3.8" +services: + tailscale: + volumes: + - ./tls/rootCA.pem:/usr/local/share/ca-certificates/rootCA.pem diff --git a/packages/grid/docker-compose.tls.yml b/packages/grid/docker-compose.tls.yml new file mode 100644 index 00000000000..7ea03b892e6 --- /dev/null +++ b/packages/grid/docker-compose.tls.yml @@ -0,0 +1,27 @@ +version: "3.8" +services: + tailscale: + ports: + - "${HTTPS_PORT}:${HTTPS_PORT}" + proxy: + command: + - "--providers.docker" + - "--providers.docker.exposedbydefault=false" + - "--providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`)" + # custom tls cert config + - "--providers.file.directory=/etc/traefik/dynamic-configurations" + - "--providers.file.watch=true" + # redirect http to https + - "--entrypoints.web.address=:81" + - "--entrypoints.web.http.redirections.entrypoint.to=websecure" + - "--entrypoints.web.http.redirections.entrypoint.scheme=https" + - "--entrypoints.vpn.address=:80" + - "--entrypoints.websecure.address=:${HTTPS_PORT}" + - "--entrypoints.websecure.http.tls=true" + # Enable the access log, with HTTP requests + - "--accesslog" + # Enable the Traefik log, for configurations and errors + - "--log" + # Enable the Dashboard and API + # - --api # admin panel + # - --api.insecure=true # admin panel no password diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index f2b195ea1c7..873f1efab0e 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -1,76 +1,88 @@ version: "3.8" services: + docker-host: + image: qoomon/docker-host + cap_add: + - net_admin + - net_raw + + tailscale: + hostname: ${DOMAIN_NAME?Variable not set} + image: "${DOCKER_IMAGE_TAILSCALE?Variable not set}:${VERSION-latest}" + build: + context: ./vpn + dockerfile: tailscale.dockerfile + environment: + - HOSTNAME=${DOMAIN_NAME?Variable not set} + - STACK_API_KEY=$STACK_API_KEY + volumes: + - tailscale-data:/var/lib/tailscale + - "/dev/net/tun:/dev/net/tun" # Required for tailscale to work + cap_add: # Required for tailscale to work + - net_admin + - sys_module + ports: + - "${HTTP_PORT}:81" + - "41641/udp" + - "4000" + # - "8080:8080" + depends_on: + - "docker-host" + proxy: restart: always - image: traefik:v2.4 + image: ${DOCKER_IMAGE_TRAEFIK?Variable not set} + # ports: + # - "${HTTP_PORT}:${HTTP_PORT}" + # - "8080:8080" networks: - "${TRAEFIK_PUBLIC_NETWORK?Variable not set}" - default + network_mode: service:tailscale volumes: - /var/run/docker.sock:/var/run/docker.sock + - ./traefik:/etc/traefik command: - # Enable Docker in Traefik, so that it reads labels from Docker services - - --providers.docker - # Add a constraint to only use services with the label for this stack - # from the env var TRAEFIK_TAG - - --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`) - # Do not expose all Docker services, only the ones explicitly exposed - - --providers.docker.exposedbydefault=false - # Enable Docker Swarm mode - - --providers.docker.swarmmode + - "--providers.docker" + - "--providers.docker.exposedbydefault=false" + - "--providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`)" + - "--entrypoints.web.address=:81" + - "--entrypoints.vpn.address=:80" + # Enable the access log, with HTTP requests - - --accesslog + - "--accesslog" # Enable the Traefik log, for configurations and errors - - --log + - "--log" # Enable the Dashboard and API - - --api - deploy: - placement: - constraints: - - node.role == manager - labels: - # Enable Traefik for this service, to make it available in the public network - - traefik.enable=true - # Use the traefik-public network (declared below) - - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set} - # Use the custom label "traefik.constraint-label=traefik-public" - # This public Traefik will only use services with this label - - traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set} - # traefik-http set up only to use the middleware to redirect to https - - traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.scheme=https - - traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.permanent=true - # Handle host with and without "www" to redirect to only one of them - # Uses environment variable DOMAIN - # To disable www redirection remove the Host() you want to discard, here and - # below for HTTPS - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`) - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.entrypoints=http - # traefik-https the actual router using HTTPS - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`) - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.entrypoints=https - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls=true - # Use the "le" (Let's Encrypt) resolver created below - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls.certresolver=le - # Define the port inside of the Docker service to use - - traefik.http.services.${STACK_NAME?Variable not set}-proxy.loadbalancer.server.port=80 - # Handle domain with and without "www" to redirect to only one - # To disable www redirection remove the next line - - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.regex=^https?://(www.)?(${DOMAIN?Variable not set})/(.*) - # Redirect a domain with www to non-www - # To disable it remove the next line - - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.replacement=https://${DOMAIN?Variable not set}/$${3} - # Redirect a domain without www to www - # To enable it remove the previous line and uncomment the next - # - traefik.http.middlewares.${STACK_NAME}-www-redirect.redirectregex.replacement=https://www.${DOMAIN}/$${3} - # Middleware to redirect www, to disable it remove the next line - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.middlewares=${STACK_NAME?Variable not set}-www-redirect - # Middleware to redirect www, and redirect HTTP to HTTPS - # to disable www redirection remove the section: ${STACK_NAME?Variable not set}-www-redirect, - - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.middlewares=${STACK_NAME?Variable not set}-www-redirect,${STACK_NAME?Variable not set}-https-redirect + # - --api # admin panel + # - --api.insecure=true # admin panel no password + + frontend: + restart: always + image: "${DOCKER_IMAGE_FRONTEND?Variable not set}:${VERSION-latest}" + profiles: + - frontend + build: + context: ./frontend + dockerfile: frontend.dockerfile + target: "grid-ui-${FRONTEND_ENV-development}" + args: + FRONTEND_ENV: ${FRONTEND_ENV-development} + environment: + - NODE_TYPE=${NODE_TYPE?Variable not set} + - VERSION=${VERSION} + - VERSION_HASH=${VERSION_HASH} + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}" + - "traefik.http.routers.${STACK_NAME?Variable not set}-frontend.rule=PathPrefix(`/`)" + - "traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80" db: restart: always image: postgres:12 + ports: + - "5432" volumes: - app-db-data:/var/lib/postgresql/data env_file: @@ -82,10 +94,12 @@ services: queue: restart: always - image: rabbitmq:3 - # Using the below image instead is required to enable the "Broker" tab in the flower UI: - # image: rabbitmq:3-management - # + image: rabbitmq:3-management + ports: + - "5672" + - "15672" + volumes: + - ./rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf backend: restart: always @@ -97,20 +111,27 @@ services: environment: - VERSION=${VERSION} - VERSION_HASH=${VERSION_HASH} - - SERVER_NAME=${DOMAIN?Variable not set} - - SERVER_HOST=https://${DOMAIN?Variable not set} - # Allow explicit env var override for tests - - SMTP_HOST=${SMTP_HOST} + - NODE_TYPE=${NODE_TYPE?Variable not set} + - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} + - STACK_API_KEY=${STACK_API_KEY} + - PORT=8001 + - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} build: context: ../ dockerfile: ./grid/backend/backend.dockerfile target: "backend" - deploy: - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`) # WARNING: this wont match /api/v1/syft/stream because of length - - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80 + volumes: + - ./backend/grid:/app/grid + - ./backend/alembic:/app/alembic + - ../syft:/app/syft + - ./data/package-cache:/root/.cache + command: /start-reload.sh + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}" + - "traefik.http.routers.${STACK_NAME?Variable not set}-backend.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" # WARNING: this wont match /api/v1/syft/stream because of length + - "traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=8001" + network_mode: service:proxy backend_stream: restart: always @@ -123,21 +144,28 @@ services: environment: - VERSION=${VERSION} - VERSION_HASH=${VERSION_HASH} - - SERVER_NAME=${DOMAIN?Variable not set} - - SERVER_HOST=https://${DOMAIN?Variable not set} - # Allow explicit env var override for tests - - SMTP_HOST=${SMTP_HOST} - command: "waitforit -address=http://backend:80/api/v1/syft/metadata -status=200 -timeout=600 -- /start.sh" + - NODE_TYPE=${NODE_TYPE?Variable not set} + - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} + - STACK_API_KEY=${STACK_API_KEY} + - PORT=8011 + - STREAM_QUEUE=1 + - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} build: context: ../ dockerfile: ./grid/backend/backend.dockerfile target: "backend" - deploy: - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-backend-stream-http.rule=PathPrefix(`/api`) && PathPrefix(`/api/v1/syft/stream`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`) # WARNING: this only matches /api/v1/syft/stream because of length - - traefik.http.services.${STACK_NAME?Variable not set}-backend-stream.loadbalancer.server.port=80 + volumes: + - ./backend/grid:/app/grid + - ./backend/alembic:/app/alembic + - ../syft:/app/syft + - ./data/package-cache:/root/.cache + command: "waitforit -address=http://localhost:8001/api/v1/syft/metadata -status=200 -timeout=600 -- /start-reload.sh" + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}" + - "traefik.http.routers.${STACK_NAME?Variable not set}-backend-stream.rule=PathPrefix(`/api`) && PathPrefix(`/api/v1/syft/stream`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" # WARNING: this only matches /api/v1/syft/stream because of length + - "traefik.http.services.${STACK_NAME?Variable not set}-backend-stream.loadbalancer.server.port=8011" + network_mode: service:proxy celeryworker: restart: always @@ -150,35 +178,53 @@ services: environment: - VERSION=${VERSION} - VERSION_HASH=${VERSION_HASH} - - SERVER_NAME=${DOMAIN?Variable not set} - - SERVER_HOST=https://${DOMAIN?Variable not set} - # Allow explicit env var override for tests - - SMTP_HOST=${SMTP_HOST?Variable not set} + - NODE_TYPE=${NODE_TYPE?Variable not set} + - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} - CELERY_WORKER=true + - RUN=celery -A grid.worker worker -l info -Q main-queue --pool=gevent -c 500 + - C_FORCE_ROOT=1 + - STACK_API_KEY=${STACK_API_KEY} + - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} + volumes: + - ./backend/grid:/app/grid + - ./backend/alembic:/app/alembic + - ../syft:/app/syft + - ./data/package-cache:/root/.cache + command: "waitforit -address=http://localhost:8001/api/v1/syft/metadata -status=200 -timeout=600 -- /worker-start.sh" build: context: ../ dockerfile: ./grid/backend/backend.dockerfile target: "backend" + network_mode: service:proxy - frontend: - restart: always - image: "${DOCKER_IMAGE_FRONTEND?Variable not set}:${VERSION-latest}" + headscale: + profiles: + - network + hostname: headscale + image: "${DOCKER_IMAGE_HEADSCALE?Variable not set}:${VERSION-latest}" build: - context: ./frontend - dockerfile: frontend.dockerfile - target: "grid-ui-${FRONTEND_ENV-development}" + context: ./vpn + dockerfile: headscale.dockerfile + volumes: + - headscale-data:/headscale/data environment: - - VERSION=${VERSION} - - VERSION_HASH=${VERSION_HASH} - deploy: - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`) - - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 + - NETWORK_NAME=omnet + - STACK_API_KEY=$STACK_API_KEY + ports: + - "4000" + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}" + - "traefik.http.routers.${STACK_NAME?Variable not set}-vpn.rule=PathPrefix(`/vpn`)" + - "traefik.http.routers.${STACK_NAME?Variable not set}-vpn.middlewares=${STACK_NAME?Variable not set}-vpn" + - "traefik.http.middlewares.${STACK_NAME?Variable not set}-vpn.stripprefix.prefixes=/vpn" + - "traefik.http.middlewares.${STACK_NAME?Variable not set}-vpn.stripprefix.forceslash=true" + - "traefik.http.services.${STACK_NAME?Variable not set}-vpn.loadbalancer.server.port=8080" volumes: app-db-data: + tailscale-data: + headscale-data: networks: traefik-public: diff --git a/packages/grid/frontend/frontend.dockerfile b/packages/grid/frontend/frontend.dockerfile index 621b0ce7ad4..f87423c17f1 100644 --- a/packages/grid/frontend/frontend.dockerfile +++ b/packages/grid/frontend/frontend.dockerfile @@ -1,14 +1,12 @@ -ARG TYPE=domain ARG FRONTEND_DEV ARG DISABLE_TELEMETRY=1 ARG PRODUCTION_DIR=/prod_app FROM node:16-alpine as init-stage -ARG TYPE=domain ARG PRODUCTION_DIR ARG DISABLE_TELEMETRY -ENV NODE_TYPE $TYPE +ENV NODE_TYPE $NODE_TYPE ENV PROD_ROOT $PRODUCTION_DIR ENV NEXT_TELEMETRY_DISABLED $DISABLE_TELEMETRY ENV NEXT_PUBLIC_API_URL=/api/v1 @@ -19,10 +17,9 @@ RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn --f COPY . . FROM node:16-alpine as grid-ui-development -ARG TYPE=domain ARG DISABLE_TELEMETRY -ENV NODE_TYPE $TYPE +ENV NODE_TYPE $NODE_TYPE ENV NEXT_TELEMETRY_DISABLED $DISABLE_TELEMETRY ENV NEXT_PUBLIC_ENVIRONMENT=development ENV NEXT_PUBLIC_API_URL=/api/v1 @@ -46,10 +43,9 @@ ENV NEXT_TELEMETRY_DISABLED $DISABLE_TELEMETRY ENV PROD_ROOT $PRODUCTION_DIR ENV NEXT_PUBLIC_ENVIRONMENT=production ENV NEXT_PUBLIC_API_URL=/api/v1 +ENV NODE_TYPE $NODE_TYPE COPY --from=build-stage $PROD_ROOT/out /usr/share/nginx/html COPY --from=build-stage $PROD_ROOT/docker/nginx.conf /etc/nginx/conf.d/default.conf COPY --from=build-stage $PROD_ROOT/docker/nginx-backend-not-found.conf /etc/nginx/extra-conf.d/backend-not-found COPY --from=build-stage $PROD_ROOT /hauuuh - - diff --git a/packages/grid/traefik/.gitignore b/packages/grid/traefik/.gitignore new file mode 100644 index 00000000000..df9128702e3 --- /dev/null +++ b/packages/grid/traefik/.gitignore @@ -0,0 +1 @@ +certs/ diff --git a/packages/grid/traefik/certs/.gitignore b/packages/grid/traefik/certs/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/grid/traefik/dynamic-configurations/certs.yaml b/packages/grid/traefik/dynamic-configurations/certs.yaml new file mode 100644 index 00000000000..172f32799a0 --- /dev/null +++ b/packages/grid/traefik/dynamic-configurations/certs.yaml @@ -0,0 +1,17 @@ +--- +# Dynamic configuration for Traefik with TLS certificates +tls: + certificates: + - certFile: /etc/traefik/certs/cert.pem + keyFile: /etc/traefik/certs/key.pem + stores: + default: + defaultCertificate: + certFile: /etc/traefik/certs/cert.pem + keyFile: /etc/traefik/certs/key.pem + # options: + # default: + # minVersion: VersionTLS13 + # cipherSuites: + # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + # sniStrict: false diff --git a/packages/grid/vpn/tailscale.dockerfile b/packages/grid/vpn/tailscale.dockerfile index 22f379ac27e..0be26ed4cb9 100644 --- a/packages/grid/vpn/tailscale.dockerfile +++ b/packages/grid/vpn/tailscale.dockerfile @@ -1,7 +1,7 @@ FROM shaynesweeney/tailscale:latest RUN --mount=type=cache,target=/var/cache/apk \ - apk add --no-cache python3 py3-pip + apk add --no-cache python3 py3-pip ca-certificates WORKDIR /tailscale COPY ./requirements.txt /tailscale/requirements.txt diff --git a/packages/grid/vpn/tailscale.sh b/packages/grid/vpn/tailscale.sh index 283111a954a..ac18ba4821f 100755 --- a/packages/grid/vpn/tailscale.sh +++ b/packages/grid/vpn/tailscale.sh @@ -1,11 +1,22 @@ #!/bin/sh +# if we have a custom dev mode rootCA.pem it will get loaded +update-ca-certificates + # iptables --list # block all traffic but port 80 on the tailscale0 interface # iptables -A INPUT -i tailscale0 -p tcp --dport 80 -j ACCEPT # iptables -A INPUT -i tailscale0 -p tcp -j REJECT # iptables -A OUTPUT -p tcp --destination-port 4000 -j DROP +# we use 81 when SSL is enabled to redirect external port 80 traffic to SSL +# however the VPN can't use SSL because the certs cant be for IPs and the traffic +# is encrypted anyway so we dont need it +iptables -A INPUT -i tailscale0 -p tcp --destination-port 81 -j REJECT +# additionally if SSL is enabled we might be using 443+ in testing to provide +# multiple different stacks, 443, 444, 446 etc however this should be blocked +# over the VPN so we dont accidentally use it somehow +iptables -A INPUT -i tailscale0 -p tcp --destination-port 443:450 -j REJECT iptables -A INPUT -i tailscale0 -p tcp --destination-port 4000 -j REJECT iptables -A INPUT -i tailscale0 -p tcp --destination-port 8001 -j REJECT iptables -A INPUT -i tailscale0 -p tcp --destination-port 8011 -j REJECT diff --git a/packages/hagrid/hagrid/art.py b/packages/hagrid/hagrid/art.py index a2a3627a377..0012b62989f 100644 --- a/packages/hagrid/hagrid/art.py +++ b/packages/hagrid/hagrid/art.py @@ -115,17 +115,3 @@ def hagrid() -> None: i = random.randint(0, 2) options[i]() hold_on_tight() - - -# patch windows to use uft-8 output -if os.name == "nt": - try: - print("Patching Windows Default Locale to use UTF-8") - # third party - import _locale - - _locale._gdl_bak = _locale._getdefaultlocale - _locale._getdefaultlocale = lambda *args: (_locale._gdl_bak()[0], "utf8") - print("Finished Patching Windows Default Locale to use UTF-8") - except Exception as e: - print(f"Failed to patch Windows Default Locale. {e}") diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 80ea02eff15..07ac0ab2c81 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -149,6 +149,8 @@ def clean(location: str) -> None: default="", type=str, ) +@click.option("--tls", is_flag=True, help="Launch with TLS configuration") +@click.option("--test", is_flag=True, help="Launch with Test configuration") def launch(args: TypeTuple[str], **kwargs: TypeDict[str, Any]) -> None: verb = get_launch_verb() try: @@ -382,6 +384,8 @@ def create_launch_cmd( if "headless" in kwargs and str_to_bool(cast(str, kwargs["headless"])): headless = True parsed_kwargs["headless"] = headless + parsed_kwargs["tls"] = bool(kwargs["tls"]) if "tls" in kwargs else False + parsed_kwargs["test"] = bool(kwargs["test"]) if "test" in kwargs else False # If the user is using docker desktop (OSX/Windows), check to make sure there's enough RAM. # If the user is using Linux this isn't an issue because Docker scales to the avaialble RAM, @@ -692,12 +696,11 @@ def create_launch_docker_cmd( print(" - TAIL: " + str(tail)) print("\n") - # cmd += " export VERSION=$(python3 VERSION)" - # cmd += " export VERSION_HASH=$(python3 VERSION hash)" envs = { "COMPOSE_DOCKER_CLI_BUILD": 1, "DOCKER_BUILDKIT": 1, - "DOMAIN_PORT": int(host_term.free_port), + "HTTP_PORT": int(host_term.free_port), + "HTTPS_PORT": int(host_term.free_port_tls), "TRAEFIK_TAG": str(tag), "DOMAIN_NAME": str(snake_name), "NODE_TYPE": str(node_type.input), @@ -705,6 +708,10 @@ def create_launch_docker_cmd( "VERSION": GRID_SRC_VERSION[0], "VERSION_HASH": GRID_SRC_VERSION[1], } + + if kwargs["test"] is True: + envs["IGNORE_TLS_ERRORS"] = "True" + cmd = "" args = [] for k, v in envs.items(): @@ -731,6 +738,11 @@ def create_launch_docker_cmd( if kwargs["headless"] is False: cmd += " --profile frontend" + cmd += " --file docker-compose.yml" + if kwargs["tls"] is True: + cmd += " --file docker-compose.tls.yml" + if kwargs["test"] is True: + cmd += " --file docker-compose.test.yml" cmd += " up" if not tail: diff --git a/packages/hagrid/hagrid/grammar.py b/packages/hagrid/hagrid/grammar.py index c8ae0d14ce9..2d6546a219c 100644 --- a/packages/hagrid/hagrid/grammar.py +++ b/packages/hagrid/hagrid/grammar.py @@ -143,6 +143,12 @@ def port(self) -> Optional[int]: def search(self) -> bool: return bool(self.parts()[2]) + @property + def port_tls(self) -> int: + if self.port == 80: + return 443 + return 444 + @property def free_port(self) -> int: if self.port is None: @@ -151,6 +157,14 @@ def free_port(self) -> int: ) return find_available_port(host="localhost", port=self.port, search=self.search) + @property + def free_port_tls(self) -> int: + if self.port_tls is None: + raise BadGrammar( + f"{type(self)} unable to check if tls port {self.port_tls} is free" + ) + return find_available_port(host="localhost", port=self.port_tls, search=True) + def parts(self) -> TypeTuple[Optional[str], Optional[int], bool]: host = None port: Optional[int] = None diff --git a/packages/hagrid/hagrid/lib.py b/packages/hagrid/hagrid/lib.py index 8c3eff1b40e..24a7477b8b7 100644 --- a/packages/hagrid/hagrid/lib.py +++ b/packages/hagrid/hagrid/lib.py @@ -7,13 +7,13 @@ import os from pathlib import Path import site +import socket import subprocess from typing import Optional from typing import Tuple # third party import git -import requests # relative from .cache import DEFAULT_BRANCH @@ -21,16 +21,12 @@ from .deps import is_windows DOCKER_ERROR = """ -Instructions for v2 beta can be found here: -You are running an old verion of docker, possibly on Linux. You need to install v2 beta. - -https://www.rockyourcode.com/how-to-install-docker-compose-v2-on-linux-2021/ - +You are running an old version of docker, possibly on Linux. You need to install v2. At the time of writing this, if you are on linux you need to run the following: -mkdir -p ~/.docker/cli-plugins -curl -sSL https://github.com/docker/compose-cli/releases/download/v2.0.0-rc.1/docker-compose-linux-amd64 \ --o ~/.docker/cli-plugins/docker-compose +DOCKER_COMPOSE_VERSION=v2.1.1 +curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 \ + -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose ALERT: you may need to run the following command to make sure you can run without sudo. @@ -191,18 +187,21 @@ def find_available_port(host: str, port: int, search: bool = False) -> int: port_available = False while not port_available: try: - requests.get("http://" + host + ":" + str(port)) - if search: - print( - str(port) - + " doesn't seem to be available... trying " - + str(port + 1) - ) - port = port + 1 - else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result_of_check = sock.connect_ex((host, port)) + + if result_of_check != 0: + port_available = True break - except requests.ConnectionError: - port_available = True + else: + if search: + port += 1 + + except Exception as e: + print(f"Failed to check port {port}. {e}") + + sock.close() + if search is False and port_available is False: error = ( f"{port} is in use, either free the port or " diff --git a/packages/syft/proto/core/tensor/party.proto b/packages/syft/proto/core/tensor/party.proto deleted file mode 100644 index 5e53a3ca1af..00000000000 --- a/packages/syft/proto/core/tensor/party.proto +++ /dev/null @@ -1,8 +0,0 @@ -syntax = "proto3"; - -package syft.core.tensor; - -message Party { - string url = 1; - uint32 port = 2; -} diff --git a/packages/syft/proto/core/tensor/share_tensor.proto b/packages/syft/proto/core/tensor/share_tensor.proto index 1cf53c8ecc4..0a9a30ce606 100644 --- a/packages/syft/proto/core/tensor/share_tensor.proto +++ b/packages/syft/proto/core/tensor/share_tensor.proto @@ -4,8 +4,6 @@ package syft.core.tensor; import "proto/lib/numpy/array.proto"; import "proto/core/common/recursive_serde.proto"; -import "proto/core/tensor/party.proto"; - message ShareTensor { oneof data { @@ -16,5 +14,5 @@ message ShareTensor { uint32 rank = 3; uint32 seed_przs = 4; bytes ring_size = 5; - repeated Party parties_info = 6; + repeated syft.core.common.RecursiveSerde parties_info = 6; } diff --git a/packages/syft/src/syft/core/node/common/client.py b/packages/syft/src/syft/core/node/common/client.py index d60263e8cca..a0a398f4e2e 100644 --- a/packages/syft/src/syft/core/node/common/client.py +++ b/packages/syft/src/syft/core/node/common/client.py @@ -18,6 +18,7 @@ import syft as sy # relative +from ....grid import GridURL from ....logger import critical from ....logger import debug from ....logger import error @@ -182,17 +183,17 @@ def join_network( raise ValueError( "join_network requires a Client object or host_or_ip string" ) + + # we are leaving the client and entering the node in a container + # any hostnames of localhost need to be converted to docker-host if client is not None: - # connection.host has a http protocol - connection_host = client.routes[0].connection.host # type: ignore - parts = connection_host.split("://") - host_or_ip = parts[1] - # if we are using localhost to connect we need to change to docker-host - # so that the domain container can connect to the host not itself - host_or_ip = str(host_or_ip).replace("localhost", "docker-host") - return self.vpn.join_network(host_or_ip=str(host_or_ip)) # type: ignore + grid_url = client.routes[0].connection.base_url.as_docker_host() # type: ignore + else: + grid_url = GridURL.from_url(str(host_or_ip)).as_docker_host() + + return self.vpn.join_network_vpn(grid_url=grid_url) # type: ignore except Exception as e: - print(f"Failed to join network with {host_or_ip}. {e}") + print(f"Failed to join network with {client} or {host_or_ip}. {e}") @property def id(self) -> UID: diff --git a/packages/syft/src/syft/core/node/common/client_manager/vpn_api.py b/packages/syft/src/syft/core/node/common/client_manager/vpn_api.py index 2aba445d1d4..daca69e5f67 100644 --- a/packages/syft/src/syft/core/node/common/client_manager/vpn_api.py +++ b/packages/syft/src/syft/core/node/common/client_manager/vpn_api.py @@ -6,6 +6,7 @@ from typing import Type # relative +from .....grid import GridURL from .....lib.python.util import upcast from ...abstract.node import AbstractNodeClient from ..action.exception_action import ExceptionMessage @@ -18,9 +19,9 @@ class VPNAPI: def __init__(self, client: AbstractNodeClient): self.client = client - def join_network(self, host_or_ip: str) -> None: + def join_network_vpn(self, grid_url: GridURL) -> None: reply = self.perform_api_request( - syft_msg=VPNJoinMessageWithReply, content={"host_or_ip": host_or_ip} + syft_msg=VPNJoinMessageWithReply, content={"grid_url": grid_url} ) logging.info(reply.payload) status = "error" @@ -30,9 +31,9 @@ def join_network(self, host_or_ip: str) -> None: pass if status == "ok": - print(f"🔌 {self.client} successfully connected to the VPN: {host_or_ip}") + print(f"🔌 {self.client} successfully connected to the VPN: {grid_url}") else: - print(f"❌ {self.client} failed to connect to the VPN: {host_or_ip}") + print(f"❌ {self.client} failed to connect to the VPN: {grid_url}") def get_status(self) -> Dict[str, Any]: reply = self.perform_api_request(syft_msg=VPNStatusMessageWithReply, content={}) diff --git a/packages/syft/src/syft/core/node/common/node.py b/packages/syft/src/syft/core/node/common/node.py index 173bee677d8..296feebc298 100644 --- a/packages/syft/src/syft/core/node/common/node.py +++ b/packages/syft/src/syft/core/node/common/node.py @@ -27,6 +27,7 @@ import syft as sy # relative +from ....grid import GridURL from ....lib import lib_ast from ....logger import debug from ....logger import error @@ -405,7 +406,8 @@ def add_route( if host_or_ip not in node_id_dict[vpn_key]: # connect and save the client - client = sy.connect(url=f"http://{host_or_ip}/api/v1") + grid_url = GridURL.from_url(host_or_ip) + client = sy.connect(url=grid_url.with_path("/api/v1")) node_id_dict[vpn_key][host_or_ip] = client self.peer_route_clients[node_id] = node_id_dict diff --git a/packages/syft/src/syft/core/node/common/node_service/association_request/association_request_service.py b/packages/syft/src/syft/core/node/common/node_service/association_request/association_request_service.py index 6b6653198ef..6605405fe2f 100644 --- a/packages/syft/src/syft/core/node/common/node_service/association_request/association_request_service.py +++ b/packages/syft/src/syft/core/node/common/node_service/association_request/association_request_service.py @@ -15,6 +15,7 @@ import syft as sy # relative +from ......grid import GridURL from ......logger import error from ......logger import info from .....common.message import ImmediateSyftMessageWithReply @@ -99,7 +100,8 @@ def send_association_request_msg( metadata["node_name"] = ( node.name if node.name else "" ) # tell the network what our name is - target_client = sy.connect(url=f"http://{msg.target}/api/v1") + grid_url = GridURL.from_url(msg.target).with_path("/api/v1") + target_client = sy.connect(url=str(grid_url)) target_msg: SignedImmediateSyftMessageWithReply = ( ReceiveAssociationRequestMessage( @@ -231,7 +233,8 @@ def respond_association_request_msg( error(f"Failed to get vpn status. {e}") # create a client to the source - source_client = sy.connect(url=f"http://{msg.source}/api/v1") + grid_url = GridURL.from_url(msg.source).with_path("/api/v1") + source_client = sy.connect(url=str(grid_url)) try: node_msg: SignedImmediateSyftMessageWithReply = ( diff --git a/packages/syft/src/syft/core/node/common/node_service/ping/ping_messages.py b/packages/syft/src/syft/core/node/common/node_service/ping/ping_messages.py index 46e6875934c..0594e854beb 100644 --- a/packages/syft/src/syft/core/node/common/node_service/ping/ping_messages.py +++ b/packages/syft/src/syft/core/node/common/node_service/ping/ping_messages.py @@ -12,6 +12,7 @@ from typing_extensions import final # relative +from ......grid import GridURL from .....common.serde.serializable import serializable from ....abstract.node import AbstractNode from ..generic_payload.messages import GenericPayloadMessage @@ -19,6 +20,21 @@ from ..generic_payload.messages import GenericPayloadReplyMessage +def grid_url_from_kwargs(kwargs: Dict[str, Any]) -> GridURL: + try: + if "host_or_ip" in kwargs: + # old way to send these messages was with host_or_ip + return GridURL.from_url(str(kwargs["host_or_ip"])) + elif "grid_url" in kwargs: + # new way is with grid_url + return kwargs["grid_url"] + else: + raise Exception("kwargs missing host_or_ip or grid_url") + except Exception as e: + print(f"Failed to get grid_url from kwargs: {kwargs}. {e}") + raise e + + @serializable(recursive_serde=True) @final class PingMessage(GenericPayloadMessage): @@ -41,11 +57,9 @@ def run( self, node: AbstractNode, verify_key: Optional[VerifyKey] = None ) -> Dict[str, Any]: try: - host_or_ip = str(self.kwargs["host_or_ip"]) - if not host_or_ip.startswith("http"): - host_or_ip = f"http://{host_or_ip}" - res = requests.get(f"{host_or_ip}/status") - return {"host_or_ip": host_or_ip, "status_code": res.status_code} + grid_url = grid_url_from_kwargs(self.kwargs) + res = requests.get(str(grid_url.with_path("/status"))) + return {"grid_url": str(grid_url), "status_code": res.status_code} except Exception: print("Failed to run ping", self.kwargs) - return {"host_or_ip": host_or_ip, "error": "Error"} + return {"grid_url": str(grid_url), "error": "Error"} diff --git a/packages/syft/src/syft/core/node/common/node_service/vpn/vpn_messages.py b/packages/syft/src/syft/core/node/common/node_service/vpn/vpn_messages.py index ec3d6beecf2..600e84d2f80 100644 --- a/packages/syft/src/syft/core/node/common/node_service/vpn/vpn_messages.py +++ b/packages/syft/src/syft/core/node/common/node_service/vpn/vpn_messages.py @@ -19,6 +19,8 @@ from typing_extensions import final # relative +from ......grid import GridURL +from ......util import verify_tls from .....common.serde.serializable import serializable from ....abstract.node import AbstractNode from ..generic_payload.messages import GenericPayloadMessage @@ -26,6 +28,21 @@ from ..generic_payload.messages import GenericPayloadReplyMessage +def grid_url_from_kwargs(kwargs: Dict[str, Any]) -> GridURL: + try: + if "host_or_ip" in kwargs: + # old way to send these messages was with host_or_ip + return GridURL.from_url(str(kwargs["host_or_ip"])) + elif "grid_url" in kwargs: + # new way is with grid_url + return kwargs["grid_url"] + else: + raise Exception("kwargs missing host_or_ip or grid_url") + except Exception as e: + print(f"Failed to get grid_url from kwargs: {kwargs}. {e}") + raise e + + @serializable(recursive_serde=True) @final class VPNConnectMessage(GenericPayloadMessage): @@ -48,15 +65,13 @@ def run( self, node: AbstractNode, verify_key: Optional[VerifyKey] = None ) -> Dict[str, Any]: try: - host_or_ip = str(self.kwargs["host_or_ip"]) - if not host_or_ip.startswith("http"): - host_or_ip = f"http://{host_or_ip}/vpn" - + grid_url = grid_url_from_kwargs(self.kwargs) + grid_url = grid_url.with_path("/vpn") vpn_auth_key = str(self.kwargs["vpn_auth_key"]) status, error = connect_with_key( tailscale_host="http://tailscale:4000", - headscale_host=host_or_ip, + headscale_host=str(grid_url), vpn_auth_key=vpn_auth_key, ) if status: @@ -119,10 +134,7 @@ def run( self, node: AbstractNode, verify_key: Optional[VerifyKey] = None ) -> Dict[str, Any]: try: - host_or_ip = str(self.kwargs["host_or_ip"]) - if not host_or_ip.startswith("http"): - host_or_ip = f"http://{host_or_ip}" - + grid_url = grid_url_from_kwargs(self.kwargs) # can't import Network due to circular imports if type(node).__name__ == "Network": # we are already in the network and could be on the blocking backend api @@ -138,7 +150,9 @@ def run( except Exception: # nosec pass else: - res = requests.post(f"{host_or_ip}/api/v1/vpn/register") + res = requests.post( + str(grid_url.with_path("/api/v1/vpn/register")), verify=verify_tls() + ) res_json = res.json() if "vpn_auth_key" not in res_json: @@ -147,7 +161,7 @@ def run( status, error = connect_with_key( tailscale_host="http://tailscale:4000", - headscale_host=f"{host_or_ip}/vpn", + headscale_host=str(grid_url.with_path("/vpn")), vpn_auth_key=res_json["vpn_auth_key"], ) diff --git a/packages/syft/src/syft/core/node/network/client.py b/packages/syft/src/syft/core/node/network/client.py index f479d46bdbf..1d89ad5a2d0 100644 --- a/packages/syft/src/syft/core/node/network/client.py +++ b/packages/syft/src/syft/core/node/network/client.py @@ -18,7 +18,6 @@ from ...io.location import Location from ...io.location import SpecificLocation from ...io.route import Route -from ..abstract.node import AbstractNodeClient from ..common.action.exception_action import ExceptionMessage from ..common.client import Client from ..common.client_manager.association_api import AssociationRequestAPI @@ -147,29 +146,6 @@ def vm(self, new_vm: Location) -> Optional[Location]: def __repr__(self) -> str: return f"<{type(self).__name__}: {self.name}>" - def join_network( - self, - client: Optional[AbstractNodeClient] = None, - host_or_ip: Optional[str] = None, - ) -> None: - # this asks for a VPN key so it must be on a public interface hence the - # client or a public host_or_ip - try: - if client is None and host_or_ip is None: - raise ValueError( - "join_network requires a Client object or host_or_ip string" - ) - if client is not None: - # connection.host has http - connection_host = client.routes[0].connection.host # type: ignore - host_or_ip = connection_host.split("://")[1] - # if we are connecting with localhost we need to change to docker-host - # so that the domain connects to the host and not the container - host_or_ip = str(host_or_ip).replace("localhost", "docker-host") - return self.vpn.join_network(host_or_ip=str(host_or_ip)) - except Exception as e: - print(f"Failed to join network with {host_or_ip} {e}") - def vpn_status(self) -> Dict[str, Any]: return self.vpn.get_status() diff --git a/packages/syft/src/syft/core/smpc/protocol/spdz/spdz.py b/packages/syft/src/syft/core/smpc/protocol/spdz/spdz.py index e0bbe805328..5b80c76bc05 100644 --- a/packages/syft/src/syft/core/smpc/protocol/spdz/spdz.py +++ b/packages/syft/src/syft/core/smpc/protocol/spdz/spdz.py @@ -16,12 +16,10 @@ # relative from .....ast.klass import get_run_class_method -from ....node.common.client import Client from ....tensor.smpc import utils from ...store import CryptoPrimitiveProvider EXPECTED_OPS = {"mul", "matmul"} -cache_clients: Dict[Client, Client] = {} if TYPE_CHECKING: # relative diff --git a/packages/syft/src/syft/core/tensor/smpc/mpc_tensor.py b/packages/syft/src/syft/core/tensor/smpc/mpc_tensor.py index 2c6a8b9e55d..05c81881b5c 100644 --- a/packages/syft/src/syft/core/tensor/smpc/mpc_tensor.py +++ b/packages/syft/src/syft/core/tensor/smpc/mpc_tensor.py @@ -23,11 +23,11 @@ from . import utils from .... import logger from ....ast.klass import get_run_class_method +from ....grid import GridURL from ...smpc.protocol.spdz import spdz from ..passthrough import PassthroughTensor # type: ignore from ..passthrough import SupportedChainType # type: ignore from ..util import implements # type: ignore -from .party import Party from .share_tensor import ShareTensor METHODS_FORWARD_ALL_SHARES = { @@ -55,7 +55,7 @@ "resize", } -PARTIES_REGISTER_CACHE: Dict[Any, Party] = {} +PARTIES_REGISTER_CACHE: Dict[Any, GridURL] = {} class MPCTensor(PassthroughTensor): @@ -153,11 +153,11 @@ def get_ring_size_from_secret( return 2 ** 32 @staticmethod - def get_parties_info(parties: Iterable[Any]) -> List[Party]: + def get_parties_info(parties: Iterable[Any]) -> List[GridURL]: # relative from ....grid.client import GridHTTPConnection - parties_info: List[Party] = [] + parties_info: List[GridURL] = [] for party in parties: connection = party.routes[0].connection if not isinstance(connection, GridHTTPConnection): @@ -167,20 +167,11 @@ def get_parties_info(parties: Iterable[Any]) -> List[Party]: + "We apologize for the inconvenience" + "We will add support for local python objects very soon." ) - party_info = PARTIES_REGISTER_CACHE.get(party, None) + base_url = PARTIES_REGISTER_CACHE.get(party, None) - if party_info is None: + if base_url is None: base_url = connection.base_url - if base_url.count(":") == 2: - url = base_url.rsplit(":", 1)[0] - port = int(base_url.rsplit(":", 1)[1].split("/")[0]) - elif base_url.count(":") == 1: - url = base_url.rsplit("/", 2)[0] - port = 80 - else: - raise ValueError(f"Invalid base url {base_url}") - party_info = Party(url, port) - PARTIES_REGISTER_CACHE[party] = party_info + PARTIES_REGISTER_CACHE[party] = base_url try: pass # We do not use sy.register, should reenable after fixing. @@ -196,7 +187,12 @@ def get_parties_info(parties: Iterable[Any]) -> List[Party]: """ """ # TODO : should modify to return same client if registered. # print("Proxy Client already User Register", e) - parties_info.append(party_info) + if base_url is not None: + parties_info.append(base_url) + else: + raise Exception( + f"Failed to get GridURL from {base_url} for party {party}." + ) return parties_info @@ -253,7 +249,7 @@ def _get_shares_from_secret( parties: List[Any], shape: Tuple[int, ...], seed_przs: int, - parties_info: List[Party], + parties_info: List[GridURL], ring_size: int, ) -> List[ShareTensor]: if utils.ispointer(secret): @@ -282,7 +278,7 @@ def _get_shares_from_remote_secret( shape: Tuple[int, ...], parties: List[Any], seed_przs: int, - parties_info: List[Party], + parties_info: List[GridURL], ring_size: int, ) -> List[ShareTensor]: shares = [] @@ -333,7 +329,7 @@ def _get_shares_from_local_secret( secret: Any, shape: Tuple[int, ...], seed_przs: int, - parties_info: List[Party], + parties_info: List[GridURL], ring_size: int = 2 ** 32, ) -> List[ShareTensor]: shares = [] diff --git a/packages/syft/src/syft/core/tensor/smpc/party.py b/packages/syft/src/syft/core/tensor/smpc/party.py deleted file mode 100644 index ea87bcc9c07..00000000000 --- a/packages/syft/src/syft/core/tensor/smpc/party.py +++ /dev/null @@ -1,51 +0,0 @@ -# stdlib -# stdlib -from typing import Any - -# third party -from google.protobuf.reflection import GeneratedProtocolMessageType - -# relative -from ....proto.core.tensor.party_pb2 import Party as Party_PB -from ...common.serde.serializable import serializable - - -@serializable() -class Party: - __slots__ = ("url", "port") - # _DOCKER_HOST: str = "http://docker-host" - - def __init__(self, url: str, port: int) -> None: - # TODO: This is not used -- it is hardcoded to docker - # GM: Probably for real life scenario we would need to change this (when we serialize) - # for a more real life scenario - self.url = url - - self.port = port - - def _object2proto(self) -> Party_PB: - return Party_PB(url=self.url, port=self.port) - - @staticmethod - def _proto2object(proto: Party_PB) -> "Party": - # TODO: If on the same machine use docker-host - if not real address - # How to distinguish? (if 127.0.0.1 and localhost we consider using docker-host?) - res = Party(url=proto.url, port=proto.port) - return res - - @staticmethod - def get_protobuf_schema() -> GeneratedProtocolMessageType: - return Party_PB - - def __hash__(self) -> int: - # TODO: Rasswanth, George, Trask this takes into consideration a hashing based on url and port - # that we login only once - # Is that sufficient? - # res_str = f"{self.url}:{self.port}" - return hash((self.url, self.port)) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Party): - return False - - return self.port == other.port and self.url == other.url diff --git a/packages/syft/src/syft/core/tensor/smpc/share_tensor.py b/packages/syft/src/syft/core/tensor/smpc/share_tensor.py index 08ad4959f1a..c7bd4ac7da6 100644 --- a/packages/syft/src/syft/core/tensor/smpc/share_tensor.py +++ b/packages/syft/src/syft/core/tensor/smpc/share_tensor.py @@ -25,13 +25,13 @@ # relative from . import utils from .... import logger +from ....grid import GridURL from ....proto.core.tensor.share_tensor_pb2 import ShareTensor as ShareTensor_PB from ...common.serde.deserialize import _deserialize as deserialize from ...common.serde.serializable import serializable from ...common.serde.serialize import _serialize as serialize from ...smpc.store.crypto_store import CryptoStore from ..passthrough import PassthroughTensor # type: ignore -from .party import Party METHODS_FORWARD_ALL_SHARES = { "repeat", @@ -80,7 +80,7 @@ }, } -CACHE_CLIENTS: Dict[Party, Any] = {} +CACHE_CLIENTS: Dict[str, Any] = {} def populate_store(*args: List[Any], **kwargs: Dict[Any, Any]) -> None: @@ -107,7 +107,7 @@ class ShareTensor(PassthroughTensor): def __init__( self, rank: int, - parties_info: List[Party], + parties_info: List[GridURL], ring_size: int, seed_przs: int = 42, clients: Optional[List[Any]] = None, @@ -135,29 +135,27 @@ def __init__( super().__init__(value) @staticmethod - def login_clients(parties_info: List[Party]) -> Any: + def login_clients(parties_info: List[GridURL]) -> Any: clients = [] for party_info in parties_info: - party_info.url = party_info.url.replace("localhost", "docker-host") - client = CACHE_CLIENTS.get(party_info, None) + # if its localhost make it docker-host otherwise no change + external_host_info = party_info.as_docker_host() + client = CACHE_CLIENTS.get(str(external_host_info), None) + if client is None: # default cache to true, here to prevent multiple logins # due to gevent monkey patching, context switch is done during # during socket connection initialization. - CACHE_CLIENTS[party_info] = True + CACHE_CLIENTS[str(external_host_info)] = True # TODO: refactor to use a guest account client = sy.login( # nosec - url=party_info.url, + url=external_host_info, email="info@openmined.org", password="changethis", - port=party_info.port, + port=external_host_info.port, verbose=False, ) - base_url = client.routes[0].connection.base_url - client.routes[0].connection.base_url = base_url.replace( # type: ignore - "localhost", "docker-host" - ) - CACHE_CLIENTS[party_info] = client + CACHE_CLIENTS[str(external_host_info)] = client clients.append(client) return clients @@ -262,7 +260,7 @@ def generate_przs( value: Any, shape: Tuple[int, ...], rank: int, - parties_info: List[Party], + parties_info: List[GridURL], ring_size: int = 2 ** 32, seed_przs: Optional[int] = None, generator_przs: Optional[Any] = None, @@ -348,7 +346,7 @@ def generate_przs_on_dp_tensor( value: Optional[Any], shape: Tuple[int], rank: int, - parties_info: List[Party], + parties_info: List[GridURL], seed_przs: int, share_wrapper: Any, ring_size: int = 2 ** 32, diff --git a/packages/syft/src/syft/grid/__init__.py b/packages/syft/src/syft/grid/__init__.py index e69de29bb2d..55b4ee2ef54 100644 --- a/packages/syft/src/syft/grid/__init__.py +++ b/packages/syft/src/syft/grid/__init__.py @@ -0,0 +1,2 @@ +# relative +from .grid_url import GridURL # noqa: F401 diff --git a/packages/syft/src/syft/grid/client/client.py b/packages/syft/src/syft/grid/client/client.py index 2c4d0739213..15e756a2beb 100644 --- a/packages/syft/src/syft/grid/client/client.py +++ b/packages/syft/src/syft/grid/client/client.py @@ -15,11 +15,13 @@ import requests # relative +from .. import GridURL from ...core.io.connection import ClientConnection from ...core.io.route import SoloRoute from ...core.node.common.client import Client from ...core.node.domain.client import DomainClient from ...core.node.network.client import NetworkClient +from ...util import verify_tls from .grid_connection import GridHTTPConnection DEFAULT_PYGRID_PORT = 80 @@ -27,14 +29,14 @@ def connect( - url: str = DEFAULT_PYGRID_ADDRESS, + url: Union[str, GridURL] = DEFAULT_PYGRID_ADDRESS, conn_type: Type[ClientConnection] = GridHTTPConnection, credentials: Dict = {}, user_key: Optional[SigningKey] = None, ) -> Client: # Use Server metadata # to build client route - conn = conn_type(url=url) # type: ignore + conn = conn_type(url=GridURL.from_url(url)) # type: ignore if credentials: metadata, _user_key = conn.login(credentials=credentials) # type: ignore @@ -78,7 +80,7 @@ def connect( def login( - url: Optional[str] = None, + url: Optional[Union[str, GridURL]] = None, port: Optional[int] = None, email: Optional[str] = None, password: Optional[str] = None, @@ -108,20 +110,22 @@ def login( port = int(input("Please specify the port of the domain you're logging into:")) # TODO: build multiple route objects and let the Client decide which one to use - if url is None: + if isinstance(url, GridURL): + grid_url = url + elif url is None: + grid_url = GridURL(host_or_ip="docker-host", port=port, path="/api/v1/status") try: - url = "http://docker-host:" + str(port) - requests.get(url) + requests.get(str(grid_url), verify=verify_tls()) except Exception: - url = "http://localhost:" + str(port) - elif port != 80: - url = url + ":" + str(port) + grid_url.host_or_ip = "localhost" + else: + grid_url = GridURL(host_or_ip=url, port=port) + + grid_url = grid_url.with_path("/api/v1") if verbose: sys.stdout.write("Connecting to " + str(url) + "...") - url += "/api/v1" - if email is None or password is None: credentials = {} logging.info( @@ -131,7 +135,7 @@ def login( credentials = {"email": email, "password": password} # connecting to domain - node = connect(url=url, credentials=credentials, conn_type=conn_type) + node = connect(url=grid_url, credentials=credentials, conn_type=conn_type) if verbose: # bit of fanciness diff --git a/packages/syft/src/syft/grid/client/grid_connection.py b/packages/syft/src/syft/grid/client/grid_connection.py index 94fff27289a..8aa90fa3e81 100644 --- a/packages/syft/src/syft/grid/client/grid_connection.py +++ b/packages/syft/src/syft/grid/client/grid_connection.py @@ -4,6 +4,7 @@ from typing import Any from typing import Dict from typing import Tuple +from typing import Union # third party from google.protobuf.reflection import GeneratedProtocolMessageType @@ -13,6 +14,7 @@ from requests_toolbelt.multipart.encoder import MultipartEncoder # relative +from .. import GridURL from ...core.common.message import ImmediateSyftMessageWithoutReply from ...core.common.message import SignedImmediateSyftMessageWithoutReply from ...core.common.message import SyftMessage @@ -20,10 +22,12 @@ from ...core.common.serde.serialize import _serialize from ...core.node.domain.enums import RequestAPIFields from ...core.node.domain.exceptions import RequestAPIException +from ...logger import debug from ...proto.core.node.common.metadata_pb2 import Metadata as Metadata_PB from ...proto.grid.connections.http_connection_pb2 import ( GridHTTPConnection as GridHTTPConnection_PB, ) +from ...util import verify_tls from ..connections.http_connection import HTTPConnection @@ -36,8 +40,10 @@ class GridHTTPConnection(HTTPConnection): # SYFT_MULTIPART_ROUTE = "/pysyft_multipart" SIZE_THRESHOLD = 20971520 # 20 MB - def __init__(self, url: str) -> None: - self.base_url = url + def __init__(self, url: Union[GridURL, str]) -> None: + self.base_url = GridURL.from_url(url) if isinstance(url, str) else url + if self.base_url is None: + raise Exception(f"Invalid GridURL. {self.base_url}") self.session_token: str = "" self.token_type: str = "'" @@ -81,9 +87,10 @@ def _send_msg(self, msg: SyftMessage) -> requests.Response: # if sys.getsizeof(msg_bytes) < GridHTTPConnection.SIZE_THRESHOLD: # if True: r = requests.post( - url=self.base_url + route, + url=str(self.base_url) + route, data=msg_bytes, headers=header, + verify=verify_tls(), ) # else: # r = self.send_streamed_messages(blob_message=msg_bytes) @@ -94,8 +101,9 @@ def _send_msg(self, msg: SyftMessage) -> requests.Response: def login(self, credentials: Dict) -> Tuple: response = requests.post( - url=self.base_url + GridHTTPConnection.LOGIN_ROUTE, + url=str(self.base_url) + GridHTTPConnection.LOGIN_ROUTE, json=credentials, + verify=verify_tls(), ) # Response @@ -128,7 +136,20 @@ def _get_metadata(self) -> Tuple: session.mount("http://", adapter) session.mount("https://", adapter) - response = session.get(self.base_url + "/syft/metadata") + metadata_url = str(self.base_url) + "/syft/metadata" + response = session.get(metadata_url, verify=verify_tls()) + + # upgrade to tls if available + try: + if response.url.startswith("https://") and self.base_url.protocol == "http": + # we got redirected to https + self.base_url = GridURL.from_url( + response.url.replace("/syft/metadata", "") + ) + debug(f"GridURL Upgraded to HTTPS. {self.base_url}") + except Exception as e: + print(f"Failed to upgrade to HTTPS. {e}") + metadata_pb = Metadata_PB() metadata_pb.ParseFromString(response.content) @@ -136,7 +157,9 @@ def _get_metadata(self) -> Tuple: def setup(self, **content: Dict[str, Any]) -> Any: response = json.loads( - requests.post(self.base_url + "/setup", json=content).text + requests.post( + str(self.base_url) + "/setup", json=content, verify=verify_tls() + ).text ) if response.get(RequestAPIFields.MESSAGE, None): return response @@ -160,7 +183,9 @@ def reset(self) -> Any: response = json.loads( requests.delete( - self.base_url + GridHTTPConnection.SYFT_ROUTE, headers=header + str(self.base_url) + GridHTTPConnection.SYFT_ROUTE, + headers=header, + verify=verify_tls(), ).text ) if response.get(RequestAPIFields.MESSAGE, None): @@ -190,7 +215,9 @@ def send_files( "file": (file_path, open(file_path, "rb"), "application/octet-stream"), } - resp = requests.post(self.base_url + route, files=files, headers=header) + resp = requests.post( + str(self.base_url) + route, files=files, headers=header, verify=verify_tls() + ) return json.loads(resp.content) @@ -209,9 +236,10 @@ def send_streamed_messages(self, blob_message: bytes) -> requests.Response: } resp = session.post( - self.base_url + GridHTTPConnection.SYFT_ROUTE_STREAM, + str(self.base_url) + GridHTTPConnection.SYFT_ROUTE_STREAM, headers=headers, data=form, + verify=verify_tls(), ) session.close() @@ -219,18 +247,18 @@ def send_streamed_messages(self, blob_message: bytes) -> requests.Response: @property def host(self) -> str: - return self.base_url.replace("/api/v1", "") + return self.base_url.base_url @staticmethod def _proto2object(proto: GridHTTPConnection_PB) -> "GridHTTPConnection": - obj = GridHTTPConnection(url=proto.base_url) + obj = GridHTTPConnection(url=GridURL.from_url(proto.base_url)) obj.session_token = proto.session_token obj.token_type = proto.token_type return obj def _object2proto(self) -> GridHTTPConnection_PB: return GridHTTPConnection_PB( - base_url=self.base_url, + base_url=str(self.base_url), session_token=self.session_token, token_type=self.token_type, ) diff --git a/packages/syft/src/syft/grid/connections/http_connection.py b/packages/syft/src/syft/grid/connections/http_connection.py index 4db276d3ede..8a26d02d3a3 100644 --- a/packages/syft/src/syft/grid/connections/http_connection.py +++ b/packages/syft/src/syft/grid/connections/http_connection.py @@ -1,10 +1,12 @@ # stdlib import json +from typing import Union # third party import requests # relative +from .. import GridURL from ...core.common.message import SignedEventualSyftMessageWithoutReply from ...core.common.message import SignedImmediateSyftMessageWithReply from ...core.common.message import SignedImmediateSyftMessageWithoutReply @@ -18,8 +20,10 @@ class HTTPConnection(ClientConnection): - def __init__(self, url: str) -> None: - self.base_url = url + def __init__(self, url: Union[str, GridURL]) -> None: + self.base_url = GridURL.from_url(url) if isinstance(url, str) else url + if self.base_url is None: + raise Exception(f"Invalid GridURL. {self.base_url}") def send_immediate_msg_with_reply( self, msg: SignedImmediateSyftMessageWithReply @@ -91,7 +95,7 @@ def _send_msg(self, msg: SyftMessage) -> requests.Response: # Perform HTTP request using base_url as a root address data_bytes: bytes = _serialize(msg, to_bytes=True) # type: ignore r = requests.post( - url=self.base_url, + url=str(self.base_url), data=data_bytes, headers={"Content-Type": "application/octet-stream"}, ) @@ -107,7 +111,7 @@ def _get_metadata(self) -> Metadata_PB: :return: returns node metadata :rtype: str of bytes """ - data: bytes = requests.get(self.base_url + "/metadata").content + data: bytes = requests.get(str(self.base_url) + "/metadata").content metadata_pb = Metadata_PB() metadata_pb.ParseFromString(data) return metadata_pb diff --git a/packages/syft/src/syft/grid/grid_url.py b/packages/syft/src/syft/grid/grid_url.py new file mode 100644 index 00000000000..c3ebe212936 --- /dev/null +++ b/packages/syft/src/syft/grid/grid_url.py @@ -0,0 +1,110 @@ +# future +from __future__ import annotations + +# stdlib +import copy +from typing import Optional +from typing import Union +from urllib.parse import urlparse + +# third party +import requests + +# relative +from ..core.common.serde.serializable import serializable +from ..util import verify_tls + + +@serializable(recursive_serde=True) +class GridURL: + __attr_allowlist__ = ["protocol", "host_or_ip", "port", "path"] + + @staticmethod + def from_url(url: Union[str, GridURL]) -> GridURL: + if isinstance(url, GridURL): + return url + try: + # urlparse doesnt handle no protocol properly + if "://" not in url: + url = "http://" + url + parts = urlparse(url) + host_or_ip_parts = parts.netloc.split(":") + # netloc is host:port + port = 80 + if len(host_or_ip_parts) > 1: + port = int(host_or_ip_parts[1]) + host_or_ip = host_or_ip_parts[0] + return GridURL( + host_or_ip=host_or_ip, path=parts.path, port=port, protocol=parts.scheme + ) + except Exception as e: + print(f"Failed to convert url: {url} to GridURL. {e}") + raise e + + def __init__( + self, + protocol: str = "http", + host_or_ip: str = "localhost", + port: Optional[int] = 80, + path: str = "", + ) -> None: + # in case a preferred port is listed but its not clear if an alternative + # port was included in the supplied host_or_ip:port combo passed in earlier + if ":" in host_or_ip: + sub_grid_url: GridURL = GridURL.from_url(host_or_ip) + host_or_ip = str(sub_grid_url.host_or_ip) # type: ignore + port = int(sub_grid_url.port) # type: ignore + protocol = str(sub_grid_url.protocol) # type: ignore + path = str(sub_grid_url.path) # type: ignore + elif port is None: + port = 80 + + self.host_or_ip = host_or_ip + self.path = path + self.port = port + self.protocol = protocol + + def with_path(self, path: str) -> GridURL: + dupe = copy.copy(self) + dupe.path = path + return dupe + + def as_docker_host(self) -> GridURL: + if self.host_or_ip != "localhost": + return self + return GridURL( + protocol=self.protocol, + host_or_ip="docker-host", + port=self.port, + path=self.path, + ) + + @property + def url(self) -> str: + return f"{self.base_url}{self.path}" + + @property + def base_url(self) -> str: + return f"{self.protocol}://{self.host_or_ip}:{self.port}" + + def to_tls(self) -> GridURL: + if self.protocol == "https": + return self + + # TODO: only ignore ssl in dev mode + r = requests.get( + self.base_url, verify=verify_tls() + ) # ignore ssl cert if its fake + new_base_url = r.url + if new_base_url.endswith("/"): + new_base_url = new_base_url[0:-1] + return GridURL.from_url(url=f"{new_base_url}{self.path}") + + def __repr__(self) -> str: + return f"<{type(self).__name__} {self.url}>" + + def __str__(self) -> str: + return self.url + + def __hash__(self) -> int: + return hash(self.__str__()) diff --git a/packages/syft/src/syft/proto/core/tensor/party_pb2.py b/packages/syft/src/syft/proto/core/tensor/party_pb2.py deleted file mode 100644 index 7e5ea2e9aae..00000000000 --- a/packages/syft/src/syft/proto/core/tensor/party_pb2.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: proto/core/tensor/party.proto -"""Generated protocol buffer code.""" -# third party -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1dproto/core/tensor/party.proto\x12\x10syft.core.tensor""\n\x05Party\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\rb\x06proto3' -) - - -_PARTY = DESCRIPTOR.message_types_by_name["Party"] -Party = _reflection.GeneratedProtocolMessageType( - "Party", - (_message.Message,), - { - "DESCRIPTOR": _PARTY, - "__module__": "proto.core.tensor.party_pb2" - # @@protoc_insertion_point(class_scope:syft.core.tensor.Party) - }, -) -_sym_db.RegisterMessage(Party) - -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _PARTY._serialized_start = 51 - _PARTY._serialized_end = 85 -# @@protoc_insertion_point(module_scope) diff --git a/packages/syft/src/syft/proto/core/tensor/share_tensor_pb2.py b/packages/syft/src/syft/proto/core/tensor/share_tensor_pb2.py index 4d596faa3b8..8f4ff27dcc8 100644 --- a/packages/syft/src/syft/proto/core/tensor/share_tensor_pb2.py +++ b/packages/syft/src/syft/proto/core/tensor/share_tensor_pb2.py @@ -18,11 +18,10 @@ from syft.proto.core.common import ( recursive_serde_pb2 as proto_dot_core_dot_common_dot_recursive__serde__pb2, ) -from syft.proto.core.tensor import party_pb2 as proto_dot_core_dot_tensor_dot_party__pb2 from syft.proto.lib.numpy import array_pb2 as proto_dot_lib_dot_numpy_dot_array__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b"\n$proto/core/tensor/share_tensor.proto\x12\x10syft.core.tensor\x1a\x1bproto/lib/numpy/array.proto\x1a'proto/core/common/recursive_serde.proto\x1a\x1dproto/core/tensor/party.proto\"\xd9\x01\n\x0bShareTensor\x12\x32\n\x06tensor\x18\x01 \x01(\x0b\x32 .syft.core.common.RecursiveSerdeH\x00\x12+\n\x05\x61rray\x18\x02 \x01(\x0b\x32\x1a.syft.lib.numpy.NumpyProtoH\x00\x12\x0c\n\x04rank\x18\x03 \x01(\r\x12\x11\n\tseed_przs\x18\x04 \x01(\r\x12\x11\n\tring_size\x18\x05 \x01(\x0c\x12-\n\x0cparties_info\x18\x06 \x03(\x0b\x32\x17.syft.core.tensor.PartyB\x06\n\x04\x64\x61tab\x06proto3" + b"\n$proto/core/tensor/share_tensor.proto\x12\x10syft.core.tensor\x1a\x1bproto/lib/numpy/array.proto\x1a'proto/core/common/recursive_serde.proto\"\xe2\x01\n\x0bShareTensor\x12\x32\n\x06tensor\x18\x01 \x01(\x0b\x32 .syft.core.common.RecursiveSerdeH\x00\x12+\n\x05\x61rray\x18\x02 \x01(\x0b\x32\x1a.syft.lib.numpy.NumpyProtoH\x00\x12\x0c\n\x04rank\x18\x03 \x01(\r\x12\x11\n\tseed_przs\x18\x04 \x01(\r\x12\x11\n\tring_size\x18\x05 \x01(\x0c\x12\x36\n\x0cparties_info\x18\x06 \x03(\x0b\x32 .syft.core.common.RecursiveSerdeB\x06\n\x04\x64\x61tab\x06proto3" ) @@ -41,6 +40,6 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _SHARETENSOR._serialized_start = 160 - _SHARETENSOR._serialized_end = 377 + _SHARETENSOR._serialized_start = 129 + _SHARETENSOR._serialized_end = 355 # @@protoc_insertion_point(module_scope) diff --git a/packages/syft/src/syft/registry.py b/packages/syft/src/syft/registry.py index a4a1ef3732a..df12e62adfb 100644 --- a/packages/syft/src/syft/registry.py +++ b/packages/syft/src/syft/registry.py @@ -13,6 +13,7 @@ # relative from . import login from .core.node.common.client import Client +from .grid import GridURL from .logger import error from .logger import warning @@ -39,10 +40,11 @@ def _repr_html_(self) -> str: def create_client(self, network: Dict[str, Any]) -> Client: try: - host_or_ip = network["host_or_ip"] port = int(network["port"]) protocol = network["protocol"] - return login(url=f"{protocol}://{host_or_ip}", port=port) + host_or_ip = network["host_or_ip"] + grid_url = GridURL(port=port, protocol=protocol, host_or_ip=host_or_ip) + return login(url=str(grid_url), port=port) except Exception as e: error(f"Failed to login with: {network}. {e}") raise e diff --git a/packages/syft/src/syft/util.py b/packages/syft/src/syft/util.py index e5c0b5da40e..90dbab0b031 100644 --- a/packages/syft/src/syft/util.py +++ b/packages/syft/src/syft/util.py @@ -241,6 +241,7 @@ def char_emoji(hex_chars: str) -> str: "distracted", "dreamy", "eager", + "eagleman", "ecstatic", "elastic", "elated", @@ -425,3 +426,19 @@ def get_root_data_path() -> Path: os.makedirs(data_dir, exist_ok=True) return data_dir + + +def str_to_bool(bool_str: Optional[str]) -> bool: + result = False + bool_str = str(bool_str).lower() + if bool_str == "true" or bool_str == "1": + result = True + return result + + +def verify_tls() -> bool: + return not str_to_bool(str(os.environ.get("IGNORE_TLS_ERRORS", "0"))) + + +def ssl_test() -> bool: + return len(os.environ.get("REQUESTS_CA_BUNDLE", "")) > 0 diff --git a/packages/syft/tests/conftest.py b/packages/syft/tests/conftest.py index 044af25f888..409c27a781d 100644 --- a/packages/syft/tests/conftest.py +++ b/packages/syft/tests/conftest.py @@ -1,6 +1,5 @@ # stdlib import logging -import os from typing import Any as TypeAny from typing import Callable as TypeCallable from typing import Dict as TypeDict @@ -159,17 +158,3 @@ def _helper_get_clients(nr_clients: int) -> TypeList[TypeAny]: return clients return _helper_get_clients - - -# patch windows to use uft-8 output -if os.name == "nt": - try: - print("Patching Windows Default Locale to use UTF-8") - # third party - import _locale - - _locale._gdl_bak = _locale._getdefaultlocale - _locale._getdefaultlocale = lambda *args: (_locale._gdl_bak()[0], "utf8") - print("Finished Patching Windows Default Locale to use UTF-8") - except Exception as e: - print(f"Failed to patch Windows Default Locale. {e}") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8f25aa65a22..4970ec3f14e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,7 +1,6 @@ """Configuration file to share fixtures across benchmarks.""" # stdlib -import os from typing import Any from typing import Callable from typing import List @@ -22,7 +21,10 @@ def login_clients() -> None: for i in range(PARTIES): try: client = sy.login( - email="info@openmined.org", password="changethis", port=(PORT + i) + email="info@openmined.org", + password="changethis", + port=(PORT + i), + verbose=False, ) clients.append(client) except Exception as e: @@ -48,17 +50,3 @@ def pytest_configure(config: _pytest.config.Config) -> None: config.addinivalue_line("markers", "k8s: kubernetes integration tests") config.addinivalue_line("markers", "e2e: end-to-end integration tests") config.addinivalue_line("markers", "security: security integration tests") - - -# patch windows to use uft-8 output -if os.name == "nt": - try: - print("Patching Windows Default Locale to use UTF-8") - # third party - import _locale - - _locale._gdl_bak = _locale._getdefaultlocale - _locale._getdefaultlocale = lambda *args: (_locale._gdl_bak()[0], "utf8") - print("Finished Patching Windows Default Locale to use UTF-8") - except Exception as e: - print(f"Failed to patch Windows Default Locale. {e}") diff --git a/tests/integration/e2e/trade_demo_smpc_adp_test.py b/tests/integration/e2e/trade_demo_smpc_adp_test.py index 2dde16d86c4..e8b1dbe368c 100644 --- a/tests/integration/e2e/trade_demo_smpc_adp_test.py +++ b/tests/integration/e2e/trade_demo_smpc_adp_test.py @@ -173,7 +173,6 @@ def test_end_to_end_smpc_adp_trade_demo() -> None: """ # the prestige 🎩 print("running the prestige 🎩") - # time.sleep(40) # TODO: should modify after implementing polling .get() public_result.block_with_timeout(40) diff --git a/tests/integration/network/connect_nodes_test.py b/tests/integration/network/connect_nodes_test.py index 3829da79ad9..fec3c40cd20 100644 --- a/tests/integration/network/connect_nodes_test.py +++ b/tests/integration/network/connect_nodes_test.py @@ -1,3 +1,6 @@ +# future +from __future__ import annotations + # third party import pytest import requests @@ -31,14 +34,18 @@ def join_to_network_python( def join_to_network_rest( email: str, password: str, port: int, network_host: str ) -> None: - url = f"http://localhost:{port}/api/v1/login" - auth_response = requests.post(url, json={"email": email, "password": password}) + grid_url = sy.grid.GridURL(port=port, path="/api/v1/login") + if sy.util.ssl_test(): + grid_url = grid_url.to_tls() + auth_response = requests.post( + grid_url.url, json={"email": email, "password": password} + ) auth = auth_response.json() # test HTTP API - url = f"http://localhost:{port}/api/v1/vpn/join/{network_host}" + grid_url.path = f"/api/v1/vpn/join/{network_host}" headers = {"Authorization": f"Bearer {auth['access_token']}"} - response = requests.post(url, headers=headers) + response = requests.post(grid_url.url, headers=headers) result = response.json() return result @@ -54,6 +61,8 @@ def run_network_tests(port: int, hostname: str, vpn_ip: str) -> None: assert response["status"] == "ok" host = response["host"] + if "ip" not in host: + print(response) assert host["ip"] == vpn_ip assert host["hostname"] == hostname assert host["os"] == "linux" @@ -64,6 +73,8 @@ def run_network_tests(port: int, hostname: str, vpn_ip: str) -> None: port=port, network_host=NETWORK_PUBLIC_HOST, ) + if "status" not in response or response["status"] != "ok": + print(response) assert response["status"] == "ok" diff --git a/tests/integration/security/check_vpn_firewall_test.py b/tests/integration/security/check_vpn_firewall_test.py index 7a26df8efe9..82d2797ddcb 100644 --- a/tests/integration/security/check_vpn_firewall_test.py +++ b/tests/integration/security/check_vpn_firewall_test.py @@ -44,6 +44,13 @@ def test_vpn_scan() -> None: allowed_ports = [80] blocked_ports = [21, 4000, 8001, 8011, 8080, 5050, 5432, 5555, 5672, 15672] + # when SSL is enabled we route 80 to 81 externally so that we can redirect to HTTPS + # however we want to leave normal port 80 available over the VPN internally + blocked_ports.append(81) # this shouldnt be available over the VPN + # SSL shouldnt be available over the VPN since IPs cant be used in certs + # in this case we might be bound to any number of ports during dev mode from 443+ + for i in range(443, 451): + blocked_ports.append(i) # run in two containers so that all IPs are scanned externally for container in containers: diff --git a/tox.ini b/tox.ini index af214ad59a0..c432ade93e5 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist = syft.test.fast syft.test.security stack.test.integration + stack.test.integration.tls stack.test.integration.windows stack.test.integration.k8s stack.test.integration.smpc @@ -96,6 +97,9 @@ allowlist_externals = grep sleep bash +setenv = + HAGRID_ART = false + PYTHONIOENCODING = utf-8 commands = pip install -e packages/hagrid docker --version @@ -108,8 +112,8 @@ commands = bash -c "docker volume rm test_network_1_tailscale-data --force || true" bash -c "docker volume rm test_network_1_headscale-data --force || true" bash -c 'HAGRID_ART=false hagrid launch test_network_1 network to docker:9081 --tail=false' - bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --tail=false' - bash -c 'HAGRID_ART=false hagrid launch test_domain_2 domain to docker:9083 --tail=false --headless=true' + bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --build=false --tail=false' + bash -c 'HAGRID_ART=false hagrid launch test_domain_2 domain to docker:9083 --build=false --tail=false --headless=true' docker ps bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q "event - compiled successfully" || true' bash -c '(docker logs test_network_1-frontend-1 -f &) | grep -q "event - compiled successfully" || true' @@ -141,6 +145,73 @@ commands = bash -c 'HAGRID_ART=false hagrid land test_domain_1' bash -c 'HAGRID_ART=false hagrid land test_domain_2' + +[testenv:stack.test.integration.tls] +description = Integration Tests for Core Stack with TLS +deps = + {[testenv:syft]deps} +changedir = {toxinidir} +allowlist_externals = + docker + grep + sleep + bash + mkcert + mkdir +setenv = + HAGRID_ART = false + PYTHONIOENCODING = utf-8 + IGNORE_TLS_ERRORS = True + CAROOT = {toxinidir}/packages/grid/tls + CERTS = {toxinidir}/packages/grid/traefik/certs +commands = + mkdir -p ./packages/grid/tls + bash -c "mkcert -cert-file={env:CERTS}/cert.pem -key-file={env:CERTS}/key.pem docker-host localhost 127.0.0.1 ::1" + ; # mkcert -install # use this if you want to test in your own browser + pip install -e packages/hagrid + docker --version + docker compose version + bash -c "docker volume rm test_domain_1_app-db-data --force || true" + bash -c "docker volume rm test_domain_2_app-db-data --force || true" + bash -c "docker volume rm test_network_1_app-db-data --force || true" + bash -c "docker volume rm test_domain_1_tailscale-data --force || true" + bash -c "docker volume rm test_domain_2_tailscale-data --force || true" + bash -c "docker volume rm test_network_1_tailscale-data --force || true" + bash -c "docker volume rm test_network_1_headscale-data --force || true" + bash -c "HAGRID_ART=false hagrid launch test_network_1 network to docker:9081 --tail=false --tls --test" + bash -c "HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --build=false --tail=false --tls --test" + bash -c "HAGRID_ART=false hagrid launch test_domain_2 domain to docker:9083 --build=false --tail=false --headless=true --tls --test" + docker ps + bash -c "(docker logs test_domain_1-frontend-1 -f &) | grep -q 'event - compiled successfully' || true" + bash -c "(docker logs test_network_1-frontend-1 -f &) | grep -q 'event - compiled successfully' || true" + bash -c "(docker logs test_domain_1-backend_stream-1 -f &) | grep -q 'Application startup complete' || true" + bash -c "(docker logs test_domain_2-backend_stream-1 -f &) | grep -q 'Application startup complete' || true" + bash -c "(docker logs test_network_1-backend_stream-1 -f &) | grep -q 'Application startup complete' || true" + + sleep 5 + + pytest tests/integration -m frontend -p no:randomly --co + bash -c "REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no" + + bash -c "docker stop test_network_1-frontend-1 || true" + bash -c "docker stop test_domain_1-frontend-1 || true" + + pytest tests/integration -m network -p no:randomly --co + bash -c "REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m network -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no" + + pytest tests/integration -m general -p no:randomly --co + bash -c "REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m general -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no" + + pytest tests/integration -m e2e -p no:randomly --co + bash -c "REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m e2e -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no" + + pytest tests/integration -m security -p no:randomly --co + bash -c "REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m security -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no" + + bash -c "HAGRID_ART=false hagrid land test_network_1" + bash -c "HAGRID_ART=false hagrid land test_domain_1" + bash -c "HAGRID_ART=false hagrid land test_domain_2" + [testenv:stack.test.integration.windows] description = Integration Tests for Core Stack deps = @@ -209,8 +280,8 @@ commands = bash -c "docker volume rm test_domain_2_tailscale-data --force || true" bash -c "docker volume rm test_domain_3_tailscale-data --force || true" bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --tail=false --headless=true' - bash -c 'HAGRID_ART=false hagrid launch test_domain_2 domain to docker:9083 --tail=false --headless=true' - bash -c 'HAGRID_ART=false hagrid launch test_domain_3 domain to docker:9084 --tail=false --headless=true' + bash -c 'HAGRID_ART=false hagrid launch test_domain_2 domain to docker:9083 --build=false --tail=false --headless=true' + bash -c 'HAGRID_ART=false hagrid launch test_domain_3 domain to docker:9084 --build=false --tail=false --headless=true' docker ps bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'