From d3b33cf9f1d4cd771d4774c5c1604ed403de49ef Mon Sep 17 00:00:00 2001 From: Florian Braun <5863788+FloThinksPi@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:20:00 +0200 Subject: [PATCH] Add Codespaces configuration With this change one can use githubs codespaces feature that will create a development environment on a vm. A click in the Github UI will create an instance. Its free for all(single instance) except cooperations. With a simple click on now ends up with a fully functional development environment. This includes: - UAA Instance - Minio S3 Blobstore - Nginx Reverse Proxy for package uploads - Mariadb and Postgresdb - A service broker with configureable behaviour (CATS Broker) - Run Configurations for VSCode and Intellij/Rubymine to start right away - Tunneling SSH from the container to localhost to be able to connect with Intellij/Rubymines new remote ssh feature At startup the DBs are set up, migrated and seeded and also all bundle packages get installed. The setup of a codespaces takes rughly 5 minutes. Once a codespace is stopped it wont get recreated at startup so startup times are nearly instant. Also changes to the environment are persisted between restarts. Only if one choses to throw away a codespace and recreate it, thats when the build times come on top of starting times and any temporary change to the environment is gone. We will use this as our main development environment for CC. In case it gets unmaintained its a simple delete of a folder that is just used by the github codespaces functionality. Also this enables super simple start on developing in CC for most endpoints (excluding diego and loggregator functionality for now). --- .../inspectionProfiles/Default_CC_NG.xml | 10 ++ .../.idea/runConfigurations/Rubocop.xml | 31 +++++ .../_Mariadb__CC-Unittests.xml | 36 +++++ .../.idea/runConfigurations/_Mariadb__CC.xml | 32 +++++ .../_Mariadb__CC_Local_Worker.xml | 36 +++++ .../runConfigurations/_Mariadb__CC_Worker.xml | 36 +++++ .../_Postgres__CC-Unittests.xml | 36 +++++ .../.idea/runConfigurations/_Postgres__CC.xml | 32 +++++ .../_Postgres__CC_Local_Worker.xml | 36 +++++ .../_Postgres__CC_Worker.xml | 36 +++++ .../runConfigurations/_template__of_RSpec.xml | 47 +++++++ .../configs/vscode/.vscode/launch.json | 112 +++++++++++++++ .../configs/vscode/.vscode/settings.json | 16 +++ .devcontainer/devcontainer.json | 54 ++++++++ .devcontainer/docker-compose.override.yml | 19 +++ .devcontainer/images/catsbroker/Dockerfile | 17 +++ .devcontainer/images/devcontainer/Dockerfile | 7 + .devcontainer/images/devcontainer/setup.sh | 59 ++++++++ .devcontainer/images/nginx/Dockerfile | 22 +++ .devcontainer/images/nginx/conf/mime.types | 96 +++++++++++++ .devcontainer/images/nginx/conf/nginx.conf | 47 +++++++ .../nginx/conf/nginx_external_endpoints.conf | 120 ++++++++++++++++ .../images/nginx/conf/public_upload.conf | 22 +++ .devcontainer/images/uaa/Dockerfile | 53 +++++++ .../images/uaa/PatchAdminOAuthClient.java | 39 ++++++ .devcontainer/scripts/codespaces_init.sh | 17 +++ .devcontainer/scripts/codespaces_start.sh | 12 ++ .../scripts/setupDevelopmentEnvironment.sh | 129 ++++++++++++++++++ .devcontainer/scripts/setupIDEs.sh | 10 ++ .github/workflows/composite/setup/action.yml | 10 +- .gitignore | 3 + Gemfile | 1 + Gemfile.lock | 9 ++ README.md | 55 ++++++++ devenv.sh | 121 ++++++++++++++++ docker-compose.yml | 124 +++++++++++++++++ 36 files changed, 1538 insertions(+), 4 deletions(-) create mode 100644 .devcontainer/configs/intellij/.idea/inspectionProfiles/Default_CC_NG.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/Rubocop.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC-Unittests.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Local_Worker.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Worker.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC-Unittests.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Local_Worker.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Worker.xml create mode 100644 .devcontainer/configs/intellij/.idea/runConfigurations/_template__of_RSpec.xml create mode 100644 .devcontainer/configs/vscode/.vscode/launch.json create mode 100644 .devcontainer/configs/vscode/.vscode/settings.json create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.override.yml create mode 100644 .devcontainer/images/catsbroker/Dockerfile create mode 100644 .devcontainer/images/devcontainer/Dockerfile create mode 100755 .devcontainer/images/devcontainer/setup.sh create mode 100644 .devcontainer/images/nginx/Dockerfile create mode 100644 .devcontainer/images/nginx/conf/mime.types create mode 100644 .devcontainer/images/nginx/conf/nginx.conf create mode 100644 .devcontainer/images/nginx/conf/nginx_external_endpoints.conf create mode 100644 .devcontainer/images/nginx/conf/public_upload.conf create mode 100644 .devcontainer/images/uaa/Dockerfile create mode 100644 .devcontainer/images/uaa/PatchAdminOAuthClient.java create mode 100755 .devcontainer/scripts/codespaces_init.sh create mode 100755 .devcontainer/scripts/codespaces_start.sh create mode 100755 .devcontainer/scripts/setupDevelopmentEnvironment.sh create mode 100755 .devcontainer/scripts/setupIDEs.sh create mode 100755 devenv.sh create mode 100644 docker-compose.yml diff --git a/.devcontainer/configs/intellij/.idea/inspectionProfiles/Default_CC_NG.xml b/.devcontainer/configs/intellij/.idea/inspectionProfiles/Default_CC_NG.xml new file mode 100644 index 00000000000..d4d25056b52 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/inspectionProfiles/Default_CC_NG.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/Rubocop.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/Rubocop.xml new file mode 100644 index 00000000000..1032cf106de --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/Rubocop.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC-Unittests.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC-Unittests.xml new file mode 100644 index 00000000000..940255d2369 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC-Unittests.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC.xml new file mode 100644 index 00000000000..3fdfc957f90 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Local_Worker.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Local_Worker.xml new file mode 100644 index 00000000000..8b8877b52bd --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Local_Worker.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Worker.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Worker.xml new file mode 100644 index 00000000000..1aa1d7c16c2 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Mariadb__CC_Worker.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC-Unittests.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC-Unittests.xml new file mode 100644 index 00000000000..fb24803cc6a --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC-Unittests.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC.xml new file mode 100644 index 00000000000..b67c471e412 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Local_Worker.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Local_Worker.xml new file mode 100644 index 00000000000..b3ec2690b85 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Local_Worker.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Worker.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Worker.xml new file mode 100644 index 00000000000..aff8ed7c6a3 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_Postgres__CC_Worker.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/intellij/.idea/runConfigurations/_template__of_RSpec.xml b/.devcontainer/configs/intellij/.idea/runConfigurations/_template__of_RSpec.xml new file mode 100644 index 00000000000..322c2eaa172 --- /dev/null +++ b/.devcontainer/configs/intellij/.idea/runConfigurations/_template__of_RSpec.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.devcontainer/configs/vscode/.vscode/launch.json b/.devcontainer/configs/vscode/.vscode/launch.json new file mode 100644 index 00000000000..c05ece4fac8 --- /dev/null +++ b/.devcontainer/configs/vscode/.vscode/launch.json @@ -0,0 +1,112 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "[Postgres] CloudController", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/cloud_controller", + "script": "-c tmp/cloud_controller.yml", + "env": { + "DB": "postgres", + "DB_CONNECTION_STRING": "postgres://postgres:supersecret@localhost:5432/ccdb" + } + }, + { + "name": "[Mariadb] CloudController", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/cloud_controller", + "script": "-c tmp/cloud_controller.yml", + "env": { + "DB": "mysql", + "DB_CONNECTION_STRING": "mysql2://root:supersecret@localhost:3306/ccdb" + } + }, + { + "name": "[Mariadb] Unittests", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/rake", + "script": "spec:all", + "env": { + "DB": "mysql", + "MYSQL_CONNECTION_PREFIX": "mysql2://root:supersecret@localhost:3306" + } + }, + { + "name": "[Postgres] Unittests", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/rake", + "script": "spec:all", + "env": { + "DB": "postgres", + "POSTGRES_CONNECTION_PREFIX": "postgres://postgres:supersecret@localhost:5432" + } + }, + { + "name": "[Mariadb] CC Worker", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/rake", + "script": "jobs:generic", + "env": { + "DB": "mysql", + "DB_CONNECTION_STRING": "mysql2://root:supersecret@localhost:3306/ccdb" + } + }, + { + "name": "[Postgres] CC Worker", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/rake", + "script": "jobs:generic", + "env": { + "DB": "postgres", + "DB_CONNECTION_STRING": "postgres://postgres:supersecret@localhost:5432/ccdb" + } + }, + { + "name": "[Mariadb] CC Local Worker", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/rake", + "script": "jobs:local", + "env": { + "DB": "mysql", + "DB_CONNECTION_STRING": "mysql2://root:supersecret@localhost:3306/ccdb" + } + }, + { + "name": "[Postgres] CC Local Worker", + "type": "rdbg", + "request": "launch", + "useBundler": true, + "cwd": "${workspaceRoot}", + "command": "bin/rake", + "script": "jobs:local", + "env": { + "DB": "postgres", + "DB_CONNECTION_STRING": "postgres://postgres:supersecret@localhost:5432/ccdb" + } + } + ] +} \ No newline at end of file diff --git a/.devcontainer/configs/vscode/.vscode/settings.json b/.devcontainer/configs/vscode/.vscode/settings.json new file mode 100644 index 00000000000..c14a0a2ae87 --- /dev/null +++ b/.devcontainer/configs/vscode/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "ruby-spec-runner.rspecEnv": { + "DB": "postgres", + "POSTGRES_CONNECTION_PREFIX": "postgres://postgres:supersecret@localhost:5432", + "MYSQL_CONNECTION_PREFIX": "mysql2://root:supersecret@localhost:3306" + }, + "ruby.rubocop.autocorrectOnSave": true, + "ruby.rubocop.configFilePath": ".rubocop.yml", + "ruby.rubocop.hideDisableSuggestions": true, + "ruby.rubocop.useBundler": true, + "ruby.rubocop.useServer": true, + "ruby.rubocop.onSave": true, + "solargraph.useBundler": false, + "editor.bracketPairColorization.enabled": true, + "editor.guides.bracketPairs": true +} \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..4fbbe631148 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,54 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +{ + "name": "Cloud Controller Dev Environment", + "hostRequirements": { + "cpus": 4, + "memory": "8gb", + "storage": "32gb" + }, + "dockerComposeFile": [ + "../docker-compose.yml", + "docker-compose.override.yml" + ], + "service": "codespace", + "runServices": [ + "postgres", + "mysql", + "uaa", + "minio", + "catsbroker", + "nginx" + ], + "workspaceFolder": "/workspace", + "initializeCommand": ".devcontainer/scripts/codespaces_init.sh", + "onCreateCommand": ".devcontainer/scripts/codespaces_start.sh", + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": {}, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-azuretools.vscode-docker", + "oderwat.indent-rainbow", + "KoichiSasada.vscode-rdbg", + "Fooo.ruby-spec-runner", + "castwide.solargraph", + "eamodio.gitlens", + "github.vscode-github-actions" + ] + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 80, + 1234, + 9292, + 9393, + 9000, + 9001, + 8080, + 3306, + 5432 + ] +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.override.yml b/.devcontainer/docker-compose.override.yml new file mode 100644 index 00000000000..a9d2ab587a9 --- /dev/null +++ b/.devcontainer/docker-compose.override.yml @@ -0,0 +1,19 @@ +version: "3.3" +services: + + # Dev Container + codespace: + container_name: codespace + build: + context: ./ + dockerfile: .devcontainer/images/devcontainer/Dockerfile + restart: unless-stopped + command: /bin/sh -c "while sleep 1000; do :; done" + volumes: + - .:/workspace:cached + - /var/run/docker.sock:/var/run/docker.sock + network_mode: host + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined diff --git a/.devcontainer/images/catsbroker/Dockerfile b/.devcontainer/images/catsbroker/Dockerfile new file mode 100644 index 00000000000..702f156818f --- /dev/null +++ b/.devcontainer/images/catsbroker/Dockerfile @@ -0,0 +1,17 @@ +FROM ruby:3.2-alpine + +RUN apk --no-cache add git make curl jq build-base \ + && git clone https://github.com/cloudfoundry/cf-acceptance-tests \ + --filter=blob:none \ + --depth=1 \ + -b $(curl -s https://api.github.com/repos/cloudfoundry/cf-acceptance-tests/releases/latest | jq -r '.tag_name') \ + --single-branch + +WORKDIR /cf-acceptance-tests/assets/service_broker +RUN git checkout $(curl -s https://api.github.com/repos/cloudfoundry/cf-acceptance-tests/releases/latest | jq -r '.tag_name') \ + && bundle install + +EXPOSE 80 + +ENTRYPOINT ["bundle"] +CMD ["exec", "rackup", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/.devcontainer/images/devcontainer/Dockerfile b/.devcontainer/images/devcontainer/Dockerfile new file mode 100644 index 00000000000..a62f108eabc --- /dev/null +++ b/.devcontainer/images/devcontainer/Dockerfile @@ -0,0 +1,7 @@ +FROM mcr.microsoft.com/vscode/devcontainers/ruby + +COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ +COPY .devcontainer/images/devcontainer/setup.sh /tmp/setup.sh +COPY .ruby-version /tmp/.ruby-version +USER vscode +RUN /tmp/setup.sh && sudo rm /tmp/setup.sh /tmp/.ruby-version \ No newline at end of file diff --git a/.devcontainer/images/devcontainer/setup.sh b/.devcontainer/images/devcontainer/setup.sh new file mode 100755 index 00000000000..64ac33c39ba --- /dev/null +++ b/.devcontainer/images/devcontainer/setup.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -Eeuo pipefail +# shellcheck disable=SC2064 +trap "pkill -P $$" EXIT + +setupAptPackages () { + # CF CLI is not available for aarch64 :( + if [[ $(uname -m) == aarch64 ]]; then + PACKAGES="postgresql-client postgresql-client-common mariadb-client ruby-dev" + else + PACKAGES="cf8-cli postgresql-client postgresql-client-common mariadb-client ruby-dev" + fi + + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + export DEBIAN_FRONTEND="noninteractive" && echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + sudo apt-get install -o Dpkg::Options::="--force-overwrite" $PACKAGES -y +} + +setupRuby () { + rbenv install $(cat /tmp/.ruby-version) + rbenv global $(cat /tmp/.ruby-version) +} + +setupRubyGems () { + gem install cf-uaac +} + +setupCredhubCli () { + set -x + wget "$(curl -s https://api.github.com/repos/cloudfoundry/credhub-cli/releases/latest | + jq -r '.assets[] | select(.name|match("credhub-linux.*")) | .browser_download_url')" -O /tmp/credhub.tar.gz + cd /tmp + sudo tar -xzf /tmp/credhub.tar.gz && sudo rm -f /tmp/credhub.tar.gz && sudo mv /tmp/credhub /usr/bin +} + +setupYqCli () { + sudo wget "$(curl -s https://api.github.com/repos/mikefarah/yq/releases/latest | + jq -r '.assets[] | select(.name|match("linux_amd64$")) | .browser_download_url')" -O /usr/bin/yq + sudo chmod +x /usr/bin/yq +} + +echo """ +export COMPOSE_DOCKER_CLI_BUILD=1 +export DOCKER_BUILDKIT=1 +""" > ~/.bashrc + +setupAptPackages +setupRuby +setupRubyGems +setupCredhubCli +setupYqCli + +# Setup User Permissions +sudo groupadd docker +sudo usermod -aG docker "vscode" + +trap "" EXIT \ No newline at end of file diff --git a/.devcontainer/images/nginx/Dockerfile b/.devcontainer/images/nginx/Dockerfile new file mode 100644 index 00000000000..889a98ca11f --- /dev/null +++ b/.devcontainer/images/nginx/Dockerfile @@ -0,0 +1,22 @@ +FROM alpine + +RUN apk --no-cache add build-base pcre-dev openssl-dev gzip curl jq zlib-dev \ + && NGINX_VERSION=$(curl -s https://nginx.org/en/download.html | grep -o -E -m 1 "nginx-[0-9]+\.[0-9]+\.[0-9]+" | cut -d'-' -f2 | head -n 1) \ + && UPLOAD_VERSION=$(curl -s https://api.github.com/repos/fdintino/nginx-upload-module/tags | jq -r '.[0].name') \ + && wget -P /tmp https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz && tar -zxvf /tmp/nginx-${NGINX_VERSION}.tar.gz -C /tmp \ + && wget -P /tmp https://github.com/fdintino/nginx-upload-module/archive/${UPLOAD_VERSION}.tar.gz && tar -zxvf /tmp/${UPLOAD_VERSION}.tar.gz -C /tmp \ + && cd /tmp/nginx-${NGINX_VERSION}\ + && ./configure \ + --add-module=/tmp/nginx-upload-module-${UPLOAD_VERSION}\ + --with-http_stub_status_module \ + && make \ + && make install\ + && rm -rf /usr/local/nginx/conf + +COPY conf /usr/local/nginx/conf + +EXPOSE 80 + +STOPSIGNAL SIGQUIT + +CMD ["/usr/local/nginx/sbin/nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/.devcontainer/images/nginx/conf/mime.types b/.devcontainer/images/nginx/conf/mime.types new file mode 100644 index 00000000000..c2302120416 --- /dev/null +++ b/.devcontainer/images/nginx/conf/mime.types @@ -0,0 +1,96 @@ +types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + + font/woff woff; + font/woff2 woff2; + + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.oasis.opendocument.graphics odg; + application/vnd.oasis.opendocument.presentation odp; + application/vnd.oasis.opendocument.spreadsheet ods; + application/vnd.oasis.opendocument.text odt; + application/vnd.openxmlformats-officedocument.presentationml.presentation + pptx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + xlsx; + application/vnd.openxmlformats-officedocument.wordprocessingml.document + docx; + application/vnd.wap.wmlc wmlc; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} \ No newline at end of file diff --git a/.devcontainer/images/nginx/conf/nginx.conf b/.devcontainer/images/nginx/conf/nginx.conf new file mode 100644 index 00000000000..727a6e9cc92 --- /dev/null +++ b/.devcontainer/images/nginx/conf/nginx.conf @@ -0,0 +1,47 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 768; +} + +http { + + ## + # Basic Settings + ## + + log_format main '$host - [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" $proxy_add_x_forwarded_for vcap_request_id:$upstream_http_x_vcap_request_id response_time:$upstream_response_time'; + + sendfile on; #enable use of sendfile() + sendfile_max_chunk 1M; #make sure not to block on fast clients reading large files + tcp_nopush on; + tcp_nodelay on; #disable nagel's algorithm + + keepalive_timeout 75 20; + + limit_req_status 429; + + client_max_body_size 15M; #already enforced upstream/but doesn't hurt. + + include mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /tmp/nginx-access.log; + error_log /tmp/nginx-error.log; + + ## + # Virtual Host Configs + ## + + upstream cloud_controller { + server host.docker.internal:3000; + } + + include nginx_external_endpoints.conf; + +} \ No newline at end of file diff --git a/.devcontainer/images/nginx/conf/nginx_external_endpoints.conf b/.devcontainer/images/nginx/conf/nginx_external_endpoints.conf new file mode 100644 index 00000000000..ee7b058df3b --- /dev/null +++ b/.devcontainer/images/nginx/conf/nginx_external_endpoints.conf @@ -0,0 +1,120 @@ +server { + listen 80; + + # Forbid access to internal endpoints + location /internal/v4/ { + return 403 'Forbidden'; + } + + # proxy and log all CC traffic + location / { + access_log /tmp/nginx-access.log main; + access_log syslog:server=127.0.0.1,severity=info,tag=vcap_nginx_access main; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real_IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_redirect off; + proxy_connect_timeout 10; + proxy_pass http://cloud_controller; + } + + + + location ~ /v2/apps/[^/]+/bits { + if ($arg_application_path) { return 422; } + + include public_upload.conf; + + upload_pass_args on; + upload_pass_form_field "^resources$"; + } + + location ~ /v2/buildpacks/.*/bits { + if ($arg_buildpack_path) { return 422; } + + include public_upload.conf; + } + + location ~ /v2/apps/[^/]+/droplet/upload { + if ($arg_droplet_path) { return 422; } + + include public_upload.conf; + } + + location ~ /v3/packages/.*/upload { + if ($arg_bits_path) { return 422; } + + upload_pass_form_field "^resources$"; + include public_upload.conf; + } + + location ~ /v3/buildpacks/.*/upload { + if ($arg_bits_path) { return 422; } + + include public_upload.conf; + } + + location ~ /v3/droplets/.*/upload { + if ($arg_bits_path) { return 422; } + + include public_upload.conf; + } + + location ~ /staging/(v3/)?(droplets|buildpack_cache)/.*/upload { + # Allow download the droplets and buildpacks + if ($request_method = GET){ + proxy_pass http://cloud_controller; + } + + # Allow large uploads + client_max_body_size 1536M; #already enforced upstream/but doesn't hurt. + + # Pass along auth header + set $auth_header $upstream_http_x_auth; + proxy_set_header Authorization $auth_header; + + # Pass altered request body to this location + upload_pass @cc_uploads; + + # Store files to this directory + upload_store /tmp/staged_droplet_uploads; + + # Allow uploaded files to be read only by user + #upload_store_access user:r; + + # Set specified fields in request body + upload_set_form_field "droplet_path" $upload_tmp_path; + + #on any error, delete uploaded files. + upload_cleanup 400-505; + } + + # Pass altered request body to a backend + location @cc_uploads { + proxy_pass http://cloud_controller; + } + + location ~ ^/internal_redirect/(.*){ + # only allow internal redirects + internal; + + set $download_url $1; + + #have to manualy pass along auth header + set $auth_header $upstream_http_x_auth; + proxy_set_header Authorization $auth_header; + + # Download the file and send it to client + proxy_pass $download_url; + } + + location /nginx_status { + stub_status on; + + access_log /tmp/nginx_status.access.log main; + + allow 127.0.0.1; + deny all; + } +} \ No newline at end of file diff --git a/.devcontainer/images/nginx/conf/public_upload.conf b/.devcontainer/images/nginx/conf/public_upload.conf new file mode 100644 index 00000000000..3781a1bfaf2 --- /dev/null +++ b/.devcontainer/images/nginx/conf/public_upload.conf @@ -0,0 +1,22 @@ +# Pass altered request body to this location +upload_pass @cc_uploads; + +# Allow large uploads +client_max_body_size 1536M; #already enforced upstream/but doesn't hurt. + +# Store files to this directory +upload_store /tmp/uploads; + +# No limit for output body forwarded to CC +upload_max_output_body_len 0; + +# Set specified fields in request body +upload_set_form_field "${upload_field_name}_name" $upload_file_name; +upload_set_form_field "${upload_field_name}_path" $upload_tmp_path; +upload_set_form_field "upload_start_time" $msec; + +#forward the following fields from existing body +upload_pass_form_field "^_method$"; + +#on any error, delete uploaded files. +upload_cleanup 400-505; \ No newline at end of file diff --git a/.devcontainer/images/uaa/Dockerfile b/.devcontainer/images/uaa/Dockerfile new file mode 100644 index 00000000000..c43b5f9c7fa --- /dev/null +++ b/.devcontainer/images/uaa/Dockerfile @@ -0,0 +1,53 @@ +# Build image +FROM openjdk:11.0-jdk AS builder + +WORKDIR /uaa + +# Patch admin client, add authority password.write +COPY PatchAdminOAuthClient.java /PatchAdminOAuthClient.java +RUN apt-get update && apt-get install jq -y \ + && git clone -b $(curl -s https://api.github.com/repos/cloudfoundry/uaa/releases/latest | jq -r '.tag_name') https://github.com/cloudfoundry/uaa.git . --recursive --depth=1 --shallow-submodules \ + && git clone -b $(curl -s https://api.github.com/repos/pivotal/credhub-release/releases/latest | jq -r '.tag_name') https://github.com/pivotal/credhub-release /credhub-release --recursive --depth=1 --shallow-submodules \ + && javac /PatchAdminOAuthClient.java -d / \ + && java -cp / PatchAdminOAuthClient uaa/src/main/webapp/WEB-INF/spring/oauth-clients.xml \ + && ./gradlew -Pversion=$(curl -s https://api.github.com/repos/cloudfoundry/uaa/releases/latest | jq -r '.tag_name') clean build -x test + +FROM mikefarah/yq:4.9.6 AS yq +# Newer versions don't work. The following error is shown but the build is not aborted: +# safelyRenameFile [ERRO] Failed copying from /tmp/temp<...> to /uaa.yml +# safelyRenameFile [ERRO] open /uaa.yml: permission denied +# safelyRenameFile [ERRO] Failed copying from /tmp/temp<...> to /credhub-uaa.yml +# safelyRenameFile [ERRO] open /credhub-uaa.yml: permission denied + +COPY --from=builder /uaa/scripts/cargo/uaa.yml /uaa.yml +COPY --from=builder /credhub-release/src/credhub/config/uaa.yml /credhub-uaa.yml + +# Adapt issuer URI +# Copy to uaa URL +# Remove jwt node with symmetric key +# Replace UAA client name in credhub config +# Concatenate modified cargo and credhub config files +RUN yq e '.issuer.uri = "http://localhost:8080"' -i /uaa.yml \ + && yq e '.uaa.url = .issuer.uri' -i /uaa.yml \ + && yq e 'del(.jwt)' -i /uaa.yml \ + && yq e '.oauth.clients.director_to_credhub = .oauth.clients.credhub_client' -i /credhub-uaa.yml \ + && yq e 'del(.oauth.clients.credhub_client)' -i /credhub-uaa.yml \ + && yq ea 'select(fi == 0) * select(fi == 1)' -i /uaa.yml /credhub-uaa.yml + +# Runtime image +FROM tomcat:9-jdk11 + +# Copy config file from yq image +COPY --from=yq /uaa.yml /uaa.yml + +# Remove pre-installed apps +RUN rm -rf /usr/local/tomcat/webapps/* + +# Install war from build image +COPY --from=builder /uaa/uaa/build/libs/cloudfoundry-identity-uaa-*.war /usr/local/tomcat/webapps/ROOT.war +COPY --from=builder /uaa/k8s/templates/log4j2.properties /log4j2.properties + +ENV JAVA_OPTS="-DLOGIN_CONFIG_URL=file:///uaa.yml -Dlogging.config=/log4j2.properties" +ENV spring_profiles=default + +EXPOSE 8080 \ No newline at end of file diff --git a/.devcontainer/images/uaa/PatchAdminOAuthClient.java b/.devcontainer/images/uaa/PatchAdminOAuthClient.java new file mode 100644 index 00000000000..70495ec0a89 --- /dev/null +++ b/.devcontainer/images/uaa/PatchAdminOAuthClient.java @@ -0,0 +1,39 @@ +import java.io.File; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +public class PatchAdminOAuthClient { + + public static void main(String[] args) throws Exception { + if (args.length != 1) { + System.err.println("Expecting exactly 1 argument."); + System.exit(1); + } + String xmlFile = args[0]; + + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new InputSource(xmlFile)); + + XPath xpath = XPathFactory.newInstance().newXPath(); + NodeList nodes = (NodeList)xpath.evaluate("//entry[@key='admin']/map/entry[@key='authorities']", doc, XPathConstants.NODESET); + if (nodes.getLength() != 1) { + System.err.println("Expecting exactly 1 matching node."); + System.exit(1); + } + Node value = nodes.item(0).getAttributes().getNamedItem("value"); + + value.setNodeValue(value.getNodeValue().concat(",password.write")); + + TransformerFactory.newInstance().newTransformer().transform(new DOMSource(doc), new StreamResult(new File(xmlFile))); + } +} \ No newline at end of file diff --git a/.devcontainer/scripts/codespaces_init.sh b/.devcontainer/scripts/codespaces_init.sh new file mode 100755 index 00000000000..c9aa8f38683 --- /dev/null +++ b/.devcontainer/scripts/codespaces_init.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -Eeuxo pipefail +set -o allexport +# shellcheck disable=SC2064 +trap "pkill -P $$" EXIT + +# Setup volume mounts +mkdir -p tmp + +# Speed up docker builds by prebuilding with buildx +docker-compose pull || tee tmp/fail & +docker buildx bake -f docker-compose.yml -f .devcontainer/docker-compose.override.yml || tee tmp/fail & + +wait $(jobs -p) +test -f tmp/fail && rm tmp/fail && exit 1 + +trap "" EXIT \ No newline at end of file diff --git a/.devcontainer/scripts/codespaces_start.sh b/.devcontainer/scripts/codespaces_start.sh new file mode 100755 index 00000000000..6934883c137 --- /dev/null +++ b/.devcontainer/scripts/codespaces_start.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -Eeuxo pipefail +# shellcheck disable=SC2064 +trap "pkill -P $$" EXIT + +# Setup IDEs +./.devcontainer/scripts/setupIDEs.sh + +# Setup DBs and CC Config File +./.devcontainer/scripts/setupDevelopmentEnvironment.sh + +trap "" EXIT \ No newline at end of file diff --git a/.devcontainer/scripts/setupDevelopmentEnvironment.sh b/.devcontainer/scripts/setupDevelopmentEnvironment.sh new file mode 100755 index 00000000000..c5309d6c1c6 --- /dev/null +++ b/.devcontainer/scripts/setupDevelopmentEnvironment.sh @@ -0,0 +1,129 @@ +#!/bin/bash +set -Eeuo pipefail +# shellcheck disable=SC2064 +trap "pkill -P $$" EXIT + +# Database Information +POSTGRES_CONNECTION_PREFIX="postgres://postgres:supersecret@localhost:5432" +MYSQL_CONNECTION_PREFIX="mysql2://root:supersecret@127.0.0.1:3306" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +setupPostgres () { + export DB="postgres" + # Parallel Test DBs + export POSTGRES_CONNECTION_PREFIX + bundle exec rake db:pick db:parallel:recreate + # Sequential Test DBs + export PGPASSWORD=supersecret + psql -U postgres -h localhost -tc "SELECT 1 FROM pg_database WHERE datname = 'cc_test'" | grep -q 1 || psql -U postgres -h localhost -c "CREATE DATABASE cc_test" + # Main DB + export DB_CONNECTION_STRING="${POSTGRES_CONNECTION_PREFIX}/ccdb" + bundle exec rake db:recreate db:migrate db:seed +} + +setupMariadb () { + export DB="mysql" + # Parallel Test DBs + export MYSQL_CONNECTION_PREFIX + bundle exec rake db:pick db:parallel:recreate + # Sequential Test DBs + mysql -h 127.0.0.1 -u root -psupersecret -e "CREATE DATABASE IF NOT EXISTS cc_test;" + # Main DB + export DB_CONNECTION_STRING="${MYSQL_CONNECTION_PREFIX}/ccdb" + bundle exec rake db:recreate db:migrate db:seed +} + +setupUAA () { + # Wait until ready + # shellcheck disable=SC2016 + timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8080/info)" != "200" ]]; do sleep 5; done' || false + + # Login + uaac target http://localhost:8080 --skip-ssl-validation + uaac token client get admin -s "adminsecret" + + # Admin User + NEW_ADMIN_USERNAME="ccadmin" + NEW_ADMIN_PASSWORD="secret" + uaac user add ${NEW_ADMIN_USERNAME} -p ${NEW_ADMIN_PASSWORD} --emails fake@example.com + uaac member add cloud_controller.admin ${NEW_ADMIN_USERNAME} + uaac member add uaa.admin ${NEW_ADMIN_USERNAME} + uaac member add scim.read ${NEW_ADMIN_USERNAME} + uaac member add scim.write ${NEW_ADMIN_USERNAME} + + # Dasboard User + uaac user add cc-service-dashboards -p some-sekret --emails fake2@example.com +} + +# Install packages +bundle config set --local with 'debug' +bundle install + +# Setup Containers +setupPostgres || tee tmp/fail & +setupMariadb || tee tmp/fail & +setupUAA || tee tmp/fail & + +# CC config +mkdir -p tmp +cp -a config/cloud_controller.yml tmp/cloud_controller.yml + +yq -i e '.external_domain="localhost"' tmp/cloud_controller.yml +yq -i e '.system_domain="localhost"' tmp/cloud_controller.yml + +yq -i e '.login.url="http://localhost:8080"' tmp/cloud_controller.yml +yq -i e '.login.enabled=true' tmp/cloud_controller.yml + +yq -i e '.nginx.use_nginx=true' tmp/cloud_controller.yml +yq -i e '.nginx.instance_socket=""' tmp/cloud_controller.yml + +yq -i e '.logging.file="tmp/cloud_controller.log"' tmp/cloud_controller.yml +yq -i e '.telemetry_log_path="tmp/cloud_controller_telemetry.log"' tmp/cloud_controller.yml +yq -i e '.directories.tmpdir="tmp"' tmp/cloud_controller.yml +yq -i e '.directories.diagnostics="tmp"' tmp/cloud_controller.yml +yq -i e '.security_event_logging.enabled=true' tmp/cloud_controller.yml +yq -i e '.security_event_logging.file="tmp/cef.log"' tmp/cloud_controller.yml + +yq -i e '.uaa.url="http://localhost:8080"' tmp/cloud_controller.yml +yq -i e '.uaa.internal_url="http://localhost:8080"' tmp/cloud_controller.yml +yq -i e '.uaa.resource_id="cloud_controller"' tmp/cloud_controller.yml +yq -i e 'del(.uaa.symmetric_secret)' tmp/cloud_controller.yml + +yq -i e '.resource_pool.fog_connection.provider="AWS"' tmp/cloud_controller.yml +yq -i e '.resource_pool.fog_connection.endpoint="http://localhost:9001"' tmp/cloud_controller.yml +yq -i e '.resource_pool.fog_connection.aws_access_key_id="minioadmin"' tmp/cloud_controller.yml +yq -i e '.resource_pool.fog_connection.aws_secret_access_key="minioadmin"' tmp/cloud_controller.yml +yq -i e '.resource_pool.fog_connection.aws_signature_version=2' tmp/cloud_controller.yml +yq -i e '.resource_pool.fog_connection.path_style=true' tmp/cloud_controller.yml + +yq -i e '.packages.fog_connection.provider="AWS"' tmp/cloud_controller.yml +yq -i e '.packages.fog_connection.endpoint="http://localhost:9001"' tmp/cloud_controller.yml +yq -i e '.packages.fog_connection.aws_access_key_id="minioadmin"' tmp/cloud_controller.yml +yq -i e '.packages.fog_connection.aws_secret_access_key="minioadmin"' tmp/cloud_controller.yml +yq -i e '.packages.fog_connection.aws_signature_version=2' tmp/cloud_controller.yml +yq -i e '.packages.fog_connection.path_style=true' tmp/cloud_controller.yml + +yq -i e '.droplets.fog_connection.provider="AWS"' tmp/cloud_controller.yml +yq -i e '.droplets.fog_connection.endpoint="http://localhost:9001"' tmp/cloud_controller.yml +yq -i e '.droplets.fog_connection.aws_access_key_id="minioadmin"' tmp/cloud_controller.yml +yq -i e '.droplets.fog_connection.aws_secret_access_key="minioadmin"' tmp/cloud_controller.yml +yq -i e '.droplets.fog_connection.aws_signature_version=2' tmp/cloud_controller.yml +yq -i e '.droplets.fog_connection.path_style=true' tmp/cloud_controller.yml + +yq -i e '.buildpacks.fog_connection.provider="AWS"' tmp/cloud_controller.yml +yq -i e '.buildpacks.fog_connection.endpoint="http://localhost:9001"' tmp/cloud_controller.yml +yq -i e '.buildpacks.fog_connection.aws_access_key_id="minioadmin"' tmp/cloud_controller.yml +yq -i e '.buildpacks.fog_connection.aws_secret_access_key="minioadmin"' tmp/cloud_controller.yml +yq -i e '.buildpacks.fog_connection.aws_signature_version=2' tmp/cloud_controller.yml +yq -i e '.buildpacks.fog_connection.path_style=true' tmp/cloud_controller.yml + +yq -i e '.cloud_controller_username_lookup_client_name="login"' tmp/cloud_controller.yml +yq -i e '.cloud_controller_username_lookup_client_secret="loginsecret"' tmp/cloud_controller.yml + +# Wait for background jobs and exit 1 if any error happened +# shellcheck disable=SC2046 +wait $(jobs -p) +test -f tmp/fail && rm tmp/fail && exit 1 + +trap "" EXIT \ No newline at end of file diff --git a/.devcontainer/scripts/setupIDEs.sh b/.devcontainer/scripts/setupIDEs.sh new file mode 100755 index 00000000000..19b9a2391b8 --- /dev/null +++ b/.devcontainer/scripts/setupIDEs.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -Eeuo pipefail +# shellcheck disable=SC2064 +trap "pkill -P $$" EXIT + +# Setup IDEs +cp -a -f .devcontainer/configs/vscode/.vscode . +cp -a -f .devcontainer/configs/intellij/.idea . + +trap "" EXIT \ No newline at end of file diff --git a/.github/workflows/composite/setup/action.yml b/.github/workflows/composite/setup/action.yml index 57f8ca581fa..9f9da86c923 100644 --- a/.github/workflows/composite/setup/action.yml +++ b/.github/workflows/composite/setup/action.yml @@ -39,11 +39,13 @@ runs: sudo wget -O /usr/local/bin/bosh https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-${{ inputs.BOSH_CLI_VERSION }}-linux-amd64 && sudo chmod +x /usr/local/bin/bosh shell: bash working-directory: ${{ inputs.WORKING_DIRECTORY }} + - name: Setup Bundler + run: | + mkdir .bundle + echo -e 'BUNDLE_WITHOUT: "development"' > .bundle/config + shell: bash + working-directory: ${{ inputs.WORKING_DIRECTORY }} - uses: ruby/setup-ruby@v1 with: bundler-cache: true working-directory: ${{ inputs.WORKING_DIRECTORY }} - - name: Install Ruby dependencies - run: bundle install - shell: bash - working-directory: ${{ inputs.WORKING_DIRECTORY }} diff --git a/.gitignore b/.gitignore index 4ad0499548b..280285e1fd8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *~ .idea +*.iml .vscode .yardoc .byebug_history @@ -20,9 +21,11 @@ doc .DS_Store log tmp +out tags test.err test.out +!/.devcontainer/**/* #DOCS diff --git a/Gemfile b/Gemfile index a3bc043a199..64a1c5fe457 100644 --- a/Gemfile +++ b/Gemfile @@ -102,4 +102,5 @@ group :development do gem 'spork', git: 'https://github.com/sporkrb/spork', ref: '224df49' # '~> 1.0rc' gem 'spring' gem 'spring-commands-rspec' + gem 'debug', '~> 1.8' end diff --git a/Gemfile.lock b/Gemfile.lock index e251a5ee43a..1ecfd21b0e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,6 +126,9 @@ GEM rexml crass (1.0.6) daemons (1.4.1) + debug (1.8.0) + irb (>= 1.5.0) + reline (>= 0.3.1) declarative (0.0.20) delayed_job (4.1.9) activesupport (>= 3.0, < 6.2) @@ -265,7 +268,10 @@ GEM httpclient (2.8.3) i18n (1.14.1) concurrent-ruby (~> 1.0) + io-console (0.6.0) ipaddress (0.8.3) + irb (1.7.4) + reline (>= 0.3.6) jaro_winkler (1.5.4) json (2.6.3) json-diff (0.4.1) @@ -385,6 +391,8 @@ GEM redis-client (0.15.0) connection_pool regexp_parser (2.8.1) + reline (0.3.8) + io-console (~> 0.5) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -557,6 +565,7 @@ DEPENDENCIES clockwork cloudfront-signer codeclimate-test-reporter (>= 1.0.8) + debug (~> 1.8) em-http-request (~> 1.1) eventmachine (~> 1.2.7) fluent-logger diff --git a/README.md b/README.md index 1dac0350b28..d6a10e3cb4e 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,61 @@ See [Diego Design Notes](https://github.com/cloudfoundry/diego-design-notes) for ## Contributing Please read the [contributors' guide](https://github.com/cloudfoundry/cloud_controller_ng/blob/main/CONTRIBUTING.md) and the [Cloud Foundry Code of Conduct](https://cloudfoundry.org/code-of-conduct/) +### Predefined Development Environment + +To commence your work in a fully equipped development environment, you have two main options: + +1. **GitHub Codespaces**: GitHub Codespaces provisions a virtual machine with essential core services, such as S3 Blobstore, Database, and NGINX. It also establishes a connection that your IDE can use (VSCode is recommended). To initiate a codespace, click on the green button within the GitHub UI(upper right corner) and select the 'Codespaces' tab. + +2. **Local Environment**: This option allows you to establish an environment on your local machine with the same core services as GitHub Codespaces, using Docker. + +A script in the project's root directory provides convenient shortcuts to set up an environment locally: + +``` +Usage: ./devenv.sh COMMAND + +Commands: + create - Setting up the development environment(containers) + start - Starting the development environment(containers), a existing fully set up set of containers must exist. + stop - Stopping but not removing the development environment(containers) + destroy - Stopping and removing the development environment(containers) + runconfigs - Copies matching run configurations for intellij and vscode into the respective folders + help - Print this help text +``` + +To run this script, ensure the following are installed on your local system: + +- Ruby (Refer to the .ruby-version file for the correct version) +- [Bundler](https://bundler.io/) +- [Docker](https://www.docker.com/) (Feature "Allow privileged port mapping" must be enabled in Avanced Options on Docker Desktop for Mac, docker must be accessable without root permissions) +- [Docker Compose](https://github.com/docker/compose) +- [PSQL CLI](https://www.postgresql.org/docs/current/app-psql.html) +- [MYSQL CLI](https://dev.mysql.com/doc/refman/8.0/en/mysql.html) +- [UAAC](https://github.com/cloudfoundry/cf-uaac) +- [yq 4+](https://github.com/mikefarah/yq) + +Upon executing `./devenv.sh create`, the necessary containers will be set up and the databases will be initialized and migrated. + + + +As an optional step, execute `./devenv.sh runconfigs` to copy predefined settings and run configurations for this project into `.vscode` and `.idea` directories for VSCode and IntelliJ/RubyMine/JetBrains IDEs. These configurations are opinionated and, hence, not provided by default, but they do offer common configurations to debug `rspecs`, `cloud_controller`, `local_worker`, and `generic_worker`. + +#### Credentials + +This Setup automatically creates a user in UAA for login in the cloud_controller, and sets Passwords for Postgres and Mysql. +In case you need them to configure them somewhere else (e.g. database visualizers): +- Postgres: postgres:supersecret@localhost:5432 +- MySQL: root:supersecret@127.0.0.1:3306 +- UAA Admin: +```bash +uaac target http://localhost:8080 --skip-ssl-validation +uaac token client get admin -s "adminsecret" +``` +- CF Admin: +```bash +cf api http://localhost +cf login -u ccadmin -p secret +``` ### Unit Tests **TLDR:** Always run `bundle exec rake` before committing diff --git a/devenv.sh b/devenv.sh new file mode 100755 index 00000000000..4715d6efe67 --- /dev/null +++ b/devenv.sh @@ -0,0 +1,121 @@ +#/bin/bash +set -eu +trap "pkill -P $$" EXIT + +# Help text +help_command() { + echo "Usage: $0 COMMAND" + echo "" + echo "Commands:" + echo " create - Setting up the development environment(containers)" + echo " start - Starting the development environment(containers), a existing fully set up set of containers must exist." + echo " stop - Stopping but not removing the development environment(containers)" + echo " destroy - Stopping and removing the development environment(containers)" + echo " runconfigs - Copies matching run configurations for intellij and vscode into the respective folders" + echo " help - Print this help text" +} + + +# Create a clean development environment +create_command(){ + docker-compose -p "" down + docker buildx bake -f docker-compose.yml & + docker-compose -p "" pull & + wait $(jobs -p) + docker-compose -p "" up -d + ./.devcontainer/scripts/setupDevelopmentEnvironment.sh +} + +# Start containers +start_command(){ + docker-compose -p "" start +} + +# Stop containers +stop_command(){ + docker-compose -p "" stop +} + +# Remove containers +destroy_command(){ + docker-compose -p "" down +} + +# Call Setup IDEs Script +runconfigs_command(){ + ./.devcontainer/scripts/setupIDEs.sh + echo """ + # In case you want the recommended extentions for VSCode(needed for debug, follow code symbols etc.), execute: + code --install-extension ms-azuretools.vscode-docker + code --install-extension oderwat.indent-rainbow + code --install-extension 2gua.rainbow-brackets + code --install-extension KoichiSasada.vscode-rdbg + code --install-extension Fooo.ruby-spec-runner + code --install-extension castwide.solargraph + code --install-extension eamodio.gitlens + code --install-extension github.vscode-github-actions + """ +} + +# Error handler +handle_error() { + echo "Error: Invalid command" + help_command + exit 1 +} + +# Handle no command specified +if [ $# -eq 0 ]; then + handle_error +fi + +# Check Prerequisites +export should_exit=0 +# Check Path Exists +for p in docker docker-compose ruby bundle mysql psql yq; do + if ! command -v "${p}" >/dev/null 2>&1; then + echo "Error: Dependency \"$p\" is not installed" && export should_exit=1 + fi +done +# Check execution as ruby might set a shim +# shellcheck disable=SC2043 +for p in uaac; do + if ! eval "${p} --version" >/dev/null 2>&1; then + echo "Error: Dependency \"$p\" is not installed" && export should_exit=1 + fi +done +if [ $should_exit != 0 ]; then + exit 1 +fi + +# Parse commands +case "$1" in + create) + echo "Setting up the development environment(containers)" + create_command + ;; + start) + echo "Starting the development environment(containers), a existing fully set up set of containers must exist." + start_command + ;; + stop) + echo "Stopping but not removing the development environment(containers)" + stop_command + ;; + destroy) + echo "Stopping and removing the development environment(containers)" + destroy_command + ;; + runconfigs) + echo "Copying matching run configurations for intellij and vscode into the respective folders" + runconfigs_command + ;; + help) + help_command + ;; + *) + handle_error + ;; +esac + +trap "" EXIT \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000000..13283bdffbe --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,124 @@ +version: "3.3" +services: + + # Postgres + postgres: + container_name: postgres + image: postgres:14 + environment: + POSTGRES_PASSWORD: supersecret + ports: + - "127.0.0.1:5432:5432" + networks: + - cc-net + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + # MySQL + mysql: + container_name: mysql + image: mysql:8 + environment: + MYSQL_ROOT_PASSWORD: supersecret + ports: + - "127.0.0.1:3306:3306" + healthcheck: + test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ] + interval: 10s + timeout: 20s + retries: 3 + networks: + - cc-net + restart: unless-stopped + + # UAA + uaa: + container_name: uaa + build: + context: .devcontainer/images/uaa + dockerfile: Dockerfile + ports: + - "127.0.0.1:8080:8080" + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8080/info" ] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cc-net + restart: unless-stopped + + # Minio S3 Blobstore + minio: + container_name: minio + image: minio/minio:latest + command: minio server --address ":9001" "/data" + ports: + - "127.0.0.1:9001:9001" + - "127.0.0.1:9000:9000" + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:9001/minio/health/live" + ] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cc-net + restart: unless-stopped + + # CATS Configurable Service Broker + catsbroker: + container_name: catsbroker + build: + context: .devcontainer/images/catsbroker + dockerfile: Dockerfile + ports: + - "127.0.0.1:9292:9292" + - "127.0.0.1:9393:80" + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:9292/v2/catalog" + ] + interval: 30s + timeout: 20s + retries: 3 + networks: + - cc-net + restart: unless-stopped + + # Nginx Reverse Proxy (For uploads e.g. Packages) + nginx: + container_name: nginx + build: + context: .devcontainer/images/nginx + dockerfile: Dockerfile + args: + ENABLED_MODULES: nginx_upload_module + ports: + - "0.0.0.0:80:80" + networks: + - cc-net + volumes: + - .devcontainer/images/nginx/conf:/usr/local/nginx/conf:ro + - ./tmp:/tmp + cap_add: + - NET_BIND_SERVICE + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + +networks: + cc-net: