From 611a735e4f43a61e6076dabeb142b42dd63845c4 Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Tue, 18 Jun 2024 16:49:04 -0700 Subject: [PATCH 1/3] add moto container to test_app --- Makefile | 3 ++- test_app/Makefile | 9 ++++++++- test_app/bin/init_moto.sh | 12 ++++++++++++ test_app/bin/seed_moto.sh | 14 ++++++++++++++ test_app/docker-compose.yml | 20 ++++++++++++++++++++ 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100755 test_app/bin/init_moto.sh create mode 100755 test_app/bin/seed_moto.sh diff --git a/Makefile b/Makefile index 2ddc373..333117b 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ codegen-tests: codegen ## Run tests .PHONY: gha-setup gha-setup: ## Set up the environment in CI docker swarm init - + touch test_app/.moto_recording .PHONY: build ## Build python packages and docker images build: @@ -87,6 +87,7 @@ dev: .PHONY: clean clean: ## Remove all build artifacts rm -rf dist + rm -rf test_app/.moto_recording $(docker_compose) down $(MAKE_TEST_APP) clean diff --git a/test_app/Makefile b/test_app/Makefile index aed147c..5f2087d 100644 --- a/test_app/Makefile +++ b/test_app/Makefile @@ -46,15 +46,22 @@ init: $(docker_compose_run) $(APP_CONTAINER) sh -c 'strawberry export-schema main:schema > /app/api/schema.graphql' sleep 5 # wait for the app to reload after having files updated. docker compose up -d + $(MAKE) seed-moto sleep 5 docker compose exec $(APP_CONTAINER) python3 -m sgqlc.introspection --exclude-deprecated --exclude-description http://localhost:9009/graphql api/schema.json +.PHONY: seed-moto +seed-moto: ## Seed the moto db by running the ../bin/seed_moto.sh script + touch .moto_recording + ./bin/seed_moto.sh + .PHONY: clean clean: ## Remove all codegen'd artifacts. rm -rf api rm -rf cerbos rm -rf support rm -rf database + rm -f .moto_recording $(docker_compose) --profile '*' down .PHONY: start @@ -81,7 +88,7 @@ restart: ## Restart the GQL service stop: ## Stop the local dev environment. $(docker_compose) --profile '*' stop -.PHONY: local-seed +.PHONY: seed seed: ## Seed the dev db with a reasonable set of starting data. $(docker_compose) exec $(APP_CONTAINER) python3 scripts/seed.py diff --git a/test_app/bin/init_moto.sh b/test_app/bin/init_moto.sh new file mode 100755 index 0000000..68d2fde --- /dev/null +++ b/test_app/bin/init_moto.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Script to initialize moto server; runs inside the motoserver container + +# Launch moto server +moto_server --host 0.0.0.0 --port $MOTO_PORT & + +# Initialize data once server is ready +sleep 1 && curl -X POST "http://localhost:${MOTO_PORT}/moto-api/recorder/replay-recording" + +# Go back to moto server +wait diff --git a/test_app/bin/seed_moto.sh b/test_app/bin/seed_moto.sh new file mode 100755 index 0000000..83155ff --- /dev/null +++ b/test_app/bin/seed_moto.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Script to seed moto server; runs outside the motoserver container for development + +aws="aws --endpoint-url=http://localhost:4000" +export AWS_ACCESS_KEY_ID=test +export AWS_SECRET_ACCESS_KEY=test +export AWS_REGION=us-west-2 +export S3_BUCKET=local-bucket + +if aws s3 ls "s3://$S3_BUCKET" 2>&1 | grep -q 'NoSuchBucket' +then +$aws s3 mb $S3_BUCKET +fi \ No newline at end of file diff --git a/test_app/docker-compose.yml b/test_app/docker-compose.yml index 52b27e9..32c81ca 100644 --- a/test_app/docker-compose.yml +++ b/test_app/docker-compose.yml @@ -1,4 +1,24 @@ services: + # Mock boto with persistence. + # To use it from the CLI: aws --endpoint-url=http://localhost:4000 s3 ls + # To reset all services without restarting the container: curl -X POST http://localhost:4000/moto-api/reset + motoserver: + image: motoserver/moto:latest + ports: + - "4000:4000" + environment: + - MOTO_PORT=4000 + - MOTO_ENABLE_RECORDING=True + - MOTO_S3_CUSTOM_ENDPOINTS=http://motoserver.platformics:4000 + - S3_IGNORE_SUBDOMAIN_BUCKETNAME=True + - MOTO_DOCKER_NETWORK_NAME=platformics + - MOTO_DOCKER_NETWORK_MODE=overlay + volumes: + - .moto_recording:/moto/moto_recording + - ./bin:/moto/bin + - "/var/run/docker.sock:/var/run/docker.sock" + entrypoint: ["/bin/bash"] + command: ["/moto/bin/init_moto.sh"] platformics-db: image: postgres:15 restart: always From 18d5849c1ce1e1cb24fadc44f17f31e51fa70d31 Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Thu, 20 Jun 2024 11:48:37 -0700 Subject: [PATCH 2/3] fix cerbos issue with Entity --- platformics/codegen/templates/cerbos/policies/entity.yaml.j2 | 5 ----- test_app/Makefile | 2 +- test_app/bin/seed_moto.sh | 2 +- test_app/tests/test_file_uploads.py | 3 ++- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/platformics/codegen/templates/cerbos/policies/entity.yaml.j2 b/platformics/codegen/templates/cerbos/policies/entity.yaml.j2 index 1053fb1..7eee7c7 100644 --- a/platformics/codegen/templates/cerbos/policies/entity.yaml.j2 +++ b/platformics/codegen/templates/cerbos/policies/entity.yaml.j2 @@ -13,11 +13,6 @@ resourcePolicy: derivedRoles: - project_member - - actions: ['view', 'update'] - effect: EFFECT_ALLOW - derivedRoles: - - single_user_owner - - actions: ['download'] effect: EFFECT_ALLOW derivedRoles: diff --git a/test_app/Makefile b/test_app/Makefile index 5f2087d..a13b6ce 100644 --- a/test_app/Makefile +++ b/test_app/Makefile @@ -51,7 +51,7 @@ init: docker compose exec $(APP_CONTAINER) python3 -m sgqlc.introspection --exclude-deprecated --exclude-description http://localhost:9009/graphql api/schema.json .PHONY: seed-moto -seed-moto: ## Seed the moto db by running the ../bin/seed_moto.sh script +seed-moto: ## Seed the moto db by running the ./bin/seed_moto.sh script touch .moto_recording ./bin/seed_moto.sh diff --git a/test_app/bin/seed_moto.sh b/test_app/bin/seed_moto.sh index 83155ff..779749c 100755 --- a/test_app/bin/seed_moto.sh +++ b/test_app/bin/seed_moto.sh @@ -11,4 +11,4 @@ export S3_BUCKET=local-bucket if aws s3 ls "s3://$S3_BUCKET" 2>&1 | grep -q 'NoSuchBucket' then $aws s3 mb $S3_BUCKET -fi \ No newline at end of file +fi diff --git a/test_app/tests/test_file_uploads.py b/test_app/tests/test_file_uploads.py index 38a79b0..fbba060 100644 --- a/test_app/tests/test_file_uploads.py +++ b/test_app/tests/test_file_uploads.py @@ -55,7 +55,8 @@ async def test_upload_process( }} }} """ - output = await gql_client.query(mutation, member_projects=member_projects) + + output = await gql_client.query(mutation, member_projects=member_projects, user_id=user_id) file_id = output["data"]["uploadFile"]["file"]["id"] credentials = output["data"]["uploadFile"]["credentials"] From 5eff20a767397ed6fc1b626277b29b427d06df0b Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Thu, 20 Jun 2024 13:19:39 -0700 Subject: [PATCH 3/3] add .moto_recording to ignore files --- .dockerignore | 1 + .gitignore | 1 + 2 files changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index af8fa31..9839db2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,3 +17,4 @@ **/.mypy_cache/ **/.ruff_cache/ **/.vscode/ +**/.moto_recording \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5f643b7..ea70c94 100644 --- a/.gitignore +++ b/.gitignore @@ -254,6 +254,7 @@ test_app/database/* test_app/cerbos/* test_app/support/* test_app/test_infra/* +test_app/.moto_recording # temp files /tmp/*