From 050543ae1bf44574218131253ce356989adc5506 Mon Sep 17 00:00:00 2001 From: Multazim Deshmukh <57723564+mdeshmu@users.noreply.github.com> Date: Wed, 11 Oct 2023 03:08:40 +0530 Subject: [PATCH] feat: migrate to docker compose v2 (#25565) --- .../workflows/superset-python-presto-hive.yml | 2 +- CONTRIBUTING.md | 12 ++++---- docker-compose.yml | 2 +- docker/README.md | 8 +++--- docs/docs/contributing/testing-locally.mdx | 8 +++--- docs/docs/databases/bigquery.mdx | 2 +- docs/docs/databases/docker-add-drivers.mdx | 10 +++---- docs/docs/databases/postgres.mdx | 2 +- docs/docs/databases/snowflake.mdx | 2 +- docs/docs/databases/timescaledb.mdx | 2 +- docs/docs/databases/yugabytedb.mdx | 2 +- docs/docs/installation/alerts-reports.mdx | 4 +-- .../installation/configuring-superset.mdx | 2 +- ...stalling-superset-using-docker-compose.mdx | 28 +++++++++---------- docs/docs/installation/upgrading-superset.mdx | 4 +-- scripts/tests/run.sh | 4 +-- tox.ini | 2 +- 17 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index 04cdf64d0b2dc..7e326759ea40e 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -132,7 +132,7 @@ jobs: run: sudo chown -R $USER:$USER /tmp/.superset - name: Start hadoop and hive if: steps.check.outcome == 'failure' - run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d + run: docker compose -f scripts/databases/hive/docker-compose.yml up -d - name: Setup Python if: steps.check.outcome == 'failure' uses: actions/setup-python@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 519487fabd498..d9e480ee95e5e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,7 @@ little bit helps, and credit will always be given. - [Build assets](#build-assets) - [Webpack dev server](#webpack-dev-server) - [Other npm commands](#other-npm-commands) - - [Docker (docker-compose)](#docker-docker-compose) + - [Docker (docker compose)](#docker-docker-compose) - [Updating NPM packages](#updating-npm-packages) - [Feature flags](#feature-flags) - [Git Hooks](#git-hooks) @@ -652,7 +652,7 @@ Alternatively, there are other NPM commands you may find useful: 1. `npm run build-dev`: build assets in development mode. 2. `npm run dev`: built dev assets in watch mode, will automatically rebuild when a file changes -#### Docker (docker-compose) +#### Docker (docker compose) See docs [here](docker/README.md) @@ -899,17 +899,17 @@ CYPRESS_BASE_URL= npm run cypress open See [`superset-frontend/cypress_build.sh`](https://github.com/apache/superset/blob/master/superset-frontend/cypress_build.sh). -As an alternative you can use docker-compose environment for testing: +As an alternative you can use docker compose environment for testing: Make sure you have added below line to your /etc/hosts file: `127.0.0.1 db` If you already have launched Docker environment please use the following command to assure a fresh database instance: -`docker-compose down -v` +`docker compose down -v` Launch environment: -`CYPRESS_CONFIG=true docker-compose up` +`CYPRESS_CONFIG=true docker compose up` It will serve backend and frontend on port 8088. @@ -976,7 +976,7 @@ superset: Start Superset as usual ```bash -docker-compose up +docker compose up ``` Install the required libraries and packages to the docker container diff --git a/docker-compose.yml b/docker-compose.yml index 39be1a3d3b75b..ae5cc15b82ce7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,7 +86,7 @@ services: # This is necessary because host and container need to have # their own, separate versions of these files. .dockerignore # does not seem to work when starting the service through - # docker-compose. + # docker compose. # # For example, node_modules may contain libs with native bindings. # Those bindings need to be compiled for each OS and the container diff --git a/docker/README.md b/docker/README.md index 380b96a5074b7..be29bbec0e20c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -46,9 +46,9 @@ Steps: 1. Create `./docker/requirements-local.txt` 2. Add your new packages -3. Rebuild docker-compose - 1. `docker-compose down -v` - 2. `docker-compose up` +3. Rebuild docker compose + 1. `docker compose down -v` + 2. `docker compose up` ## Initializing Database @@ -56,7 +56,7 @@ The database will initialize itself upon startup via the init container ([`super ## Normal Operation -To run the container, simply run: `docker-compose up` +To run the container, simply run: `docker compose up` After waiting several minutes for Superset initialization to finish, you can open a browser and view [`http://localhost:8088`](http://localhost:8088) to start your journey. diff --git a/docs/docs/contributing/testing-locally.mdx b/docs/docs/contributing/testing-locally.mdx index f85e87b1fa3f8..8cf1effb5d938 100644 --- a/docs/docs/contributing/testing-locally.mdx +++ b/docs/docs/contributing/testing-locally.mdx @@ -126,17 +126,17 @@ CYPRESS_BASE_URL= npm run cypress open See [`superset-frontend/cypress_build.sh`](https://github.com/apache/superset/blob/master/superset-frontend/cypress_build.sh). -As an alternative you can use docker-compose environment for testing: +As an alternative you can use docker compose environment for testing: Make sure you have added below line to your /etc/hosts file: `127.0.0.1 db` If you already have launched Docker environment please use the following command to assure a fresh database instance: -`docker-compose down -v` +`docker compose down -v` Launch environment: -`CYPRESS_CONFIG=true docker-compose up` +`CYPRESS_CONFIG=true docker compose up` It will serve backend and frontend on port 8088. @@ -176,7 +176,7 @@ superset: Start Superset as usual ```bash -docker-compose up +docker compose up ``` Install the required libraries and packages to the docker container diff --git a/docs/docs/databases/bigquery.mdx b/docs/docs/databases/bigquery.mdx index e42aa46bfb52e..6d0c36ee98e17 100644 --- a/docs/docs/databases/bigquery.mdx +++ b/docs/docs/databases/bigquery.mdx @@ -13,7 +13,7 @@ The recommended connector library for BigQuery is ### Install BigQuery Driver Follow the steps [here](/docs/databases/docker-add-drivers) about how to -install new database drivers when setting up Superset locally via docker-compose. +install new database drivers when setting up Superset locally via docker compose. ``` echo "sqlalchemy-bigquery" >> ./docker/requirements-local.txt diff --git a/docs/docs/databases/docker-add-drivers.mdx b/docs/docs/databases/docker-add-drivers.mdx index cfb504b5e474e..03a971a979efc 100644 --- a/docs/docs/databases/docker-add-drivers.mdx +++ b/docs/docs/databases/docker-add-drivers.mdx @@ -8,7 +8,7 @@ version: 1 ## Adding New Database Drivers in Docker Superset requires a Python database driver to be installed for each additional type of database you -want to connect to. When setting up Superset locally via `docker-compose`, the drivers and packages +want to connect to. When setting up Superset locally via `docker compose`, the drivers and packages contained in [requirements.txt](https://github.com/apache/superset/blob/master/requirements.txt) and [requirements-dev.txt](https://github.com/apache/superset/blob/master/requirements-dev.txt) @@ -50,20 +50,20 @@ echo "mysqlclient" >> ./docker/requirements-local.txt Rebuild your local image with the new driver baked in: ``` -docker-compose build --force-rm +docker compose build --force-rm ``` After the rebuild of the Docker images is complete (which may take a few minutes) you can relaunch using the following command: ``` -docker-compose up +docker compose up ``` The other option is to start Superset via Docker Compose is using the recipe in `docker-compose-non-dev.yml`, which will use pre-built frontend assets and skip the building of front-end assets: ``` -docker-compose -f docker-compose-non-dev.yml pull -docker-compose -f docker-compose-non-dev.yml up +docker compose -f docker-compose-non-dev.yml pull +docker compose -f docker-compose-non-dev.yml up ``` ### 3. Connect to MySQL diff --git a/docs/docs/databases/postgres.mdx b/docs/docs/databases/postgres.mdx index ff5fbc87c1277..9c3530c9d9010 100644 --- a/docs/docs/databases/postgres.mdx +++ b/docs/docs/databases/postgres.mdx @@ -7,7 +7,7 @@ version: 1 ## Postgres -Note that, if you're using docker-compose, the Postgres connector library [psycopg2](https://www.psycopg.org/docs/) +Note that, if you're using docker compose, the Postgres connector library [psycopg2](https://www.psycopg.org/docs/) comes out of the box with Superset. Postgres sample connection parameters: diff --git a/docs/docs/databases/snowflake.mdx b/docs/docs/databases/snowflake.mdx index b4ce74af7e59c..0303630087360 100644 --- a/docs/docs/databases/snowflake.mdx +++ b/docs/docs/databases/snowflake.mdx @@ -10,7 +10,7 @@ version: 1 ### Install Snowflake Driver Follow the steps [here](/docs/databases/docker-add-drivers) about how to -install new database drivers when setting up Superset locally via docker-compose. +install new database drivers when setting up Superset locally via docker compose. ``` echo "snowflake-sqlalchemy" >> ./docker/requirements-local.txt diff --git a/docs/docs/databases/timescaledb.mdx b/docs/docs/databases/timescaledb.mdx index 2ab93e68b43ff..7b85cbe1db6ad 100644 --- a/docs/docs/databases/timescaledb.mdx +++ b/docs/docs/databases/timescaledb.mdx @@ -9,7 +9,7 @@ version: 1 [TimescaleDB](https://www.timescale.com) is the open-source relational database for time-series and analytics to build powerful data-intensive applications. TimescaleDB is a PostgreSQL extension, and you can use the standard PostgreSQL connector library, [psycopg2](https://www.psycopg.org/docs/), to connect to the database. -If you're using docker-compose, psycopg2 comes out of the box with Superset. +If you're using docker compose, psycopg2 comes out of the box with Superset. TimescaleDB sample connection parameters: diff --git a/docs/docs/databases/yugabytedb.mdx b/docs/docs/databases/yugabytedb.mdx index fd60010d930e0..8c3d40ac29504 100644 --- a/docs/docs/databases/yugabytedb.mdx +++ b/docs/docs/databases/yugabytedb.mdx @@ -9,7 +9,7 @@ version: 1 [YugabyteDB](https://www.yugabyte.com/) is a distributed SQL database built on top of PostgreSQL. -Note that, if you're using docker-compose, the +Note that, if you're using docker compose, the Postgres connector library [psycopg2](https://www.psycopg.org/docs/) comes out of the box with Superset. diff --git a/docs/docs/installation/alerts-reports.mdx b/docs/docs/installation/alerts-reports.mdx index ff5dad6ccfb64..41ee19a539906 100644 --- a/docs/docs/installation/alerts-reports.mdx +++ b/docs/docs/installation/alerts-reports.mdx @@ -41,7 +41,7 @@ Note: All the components required (Firefox headless browser, Redis, Postgres db, All you need to do is add the required config variables described in this guide (See `Detailed Config`). If you are running a non-dev docker image, e.g., a stable release like `apache/superset:2.0.1`, that image does not include a headless browser. Only the `superset_worker` container needs this headless browser to browse to the target chart or dashboard. -You can either install and configure the headless browser - see "Custom Dockerfile" section below - or when deploying via `docker-compose`, modify your `docker-compose.yml` file to use a dev image for the worker container and a stable release image for the `superset_app` container. +You can either install and configure the headless browser - see "Custom Dockerfile" section below - or when deploying via `docker compose`, modify your `docker-compose.yml` file to use a dev image for the worker container and a stable release image for the `superset_app` container. *Note*: In this context, a "dev image" is the same application software as its corresponding non-dev image, just bundled with additional tools. So an image like `2.0.1-dev` is identical to `2.0.1` when it comes to stability, functionality, and running in production. The actual "in-development" versions of Superset - cutting-edge and unstable - are not tagged with version numbers on Docker Hub and will display version `0.0.0-dev` within the Superset UI. @@ -68,7 +68,7 @@ Note: when you configure an alert or a report, the Slack channel list takes chan - You must have a `celery beat` pod running. If you're using the chart included in the GitHub repository under [helm/superset](https://github.com/apache/superset/tree/master/helm/superset), you need to put `supersetCeleryBeat.enabled = true` in your values override. - You can see the dedicated docs about [Kubernetes installation](/docs/installation/running-on-kubernetes) for more generic details. -#### Docker-compose specific +#### Docker Compose specific ##### You must have in your `docker-compose.yml` diff --git a/docs/docs/installation/configuring-superset.mdx b/docs/docs/installation/configuring-superset.mdx index 3940ff4e5fea5..e970cdc4224fb 100644 --- a/docs/docs/installation/configuring-superset.mdx +++ b/docs/docs/installation/configuring-superset.mdx @@ -10,7 +10,7 @@ version: 1 ### Configuration To configure your application, you need to create a file `superset_config.py` and add it to your -`PYTHONPATH`. If your application was installed using docker-compose an alternative configuration is required. See [https://github.com/apache/superset/tree/master/docker#readme](https://github.com/apache/superset/tree/master/docker#readme) for details. +`PYTHONPATH`. If your application was installed using docker compose an alternative configuration is required. See [https://github.com/apache/superset/tree/master/docker#readme](https://github.com/apache/superset/tree/master/docker#readme) for details. The following is an example of just a few of the parameters you can set in your `superset_config.py` file: ``` diff --git a/docs/docs/installation/installing-superset-using-docker-compose.mdx b/docs/docs/installation/installing-superset-using-docker-compose.mdx index 2382e899875ce..4611466ca30f7 100644 --- a/docs/docs/installation/installing-superset-using-docker-compose.mdx +++ b/docs/docs/installation/installing-superset-using-docker-compose.mdx @@ -16,7 +16,7 @@ below. **Mac OSX** [Install Docker for Mac](https://docs.docker.com/docker-for-mac/install/), which includes the Docker -engine and a recent version of `docker-compose` out of the box. +engine and a recent version of `docker compose` out of the box. Once you have Docker for Mac installed, open up the preferences pane for Docker, go to the "Resources" section and increase the allocated memory to 6GB. With only the 2GB of RAM allocated by @@ -25,9 +25,9 @@ default, Superset will fail to start. **Linux** [Install Docker on Linux](https://docs.docker.com/engine/install/) by following Docker’s -instructions for whichever flavor of Linux suits you. Because `docker-compose` is not installed as +instructions for whichever flavor of Linux suits you. Because `docker compose` is not installed as part of the base Docker installation on Linux, once you have a working engine, follow the -[docker-compose installation instructions](https://docs.docker.com/compose/install/) for Linux. +[docker compose installation instructions](https://docs.docker.com/compose/install/) for Linux. **Windows** @@ -69,18 +69,18 @@ When running in development mode the `superset-node` container needs to finish b When working on master branch, run the following commands to run `production` mode using `docker compose`: ```bash -docker-compose -f docker-compose-non-dev.yml pull -docker-compose -f docker-compose-non-dev.yml up +docker compose -f docker-compose-non-dev.yml pull +docker compose -f docker-compose-non-dev.yml up ``` Alternatively, you can also run a specific version of Superset by first checking out -the branch/tag, and then starting `docker-compose` with the `TAG` variable. +the branch/tag, and then starting `docker compose` with the `TAG` variable. For example, to run the 3.0.0 version, run the following commands on Linux-based systems: ```bash git checkout 3.0.0 -TAG=3.0.0 docker-compose -f docker-compose-non-dev.yml pull -TAG=3.0.0 docker-compose -f docker-compose-non-dev.yml up +TAG=3.0.0 docker compose -f docker-compose-non-dev.yml pull +TAG=3.0.0 docker compose -f docker-compose-non-dev.yml up ``` If you are using Docker Desktop for Windows then run the following commands: @@ -88,8 +88,8 @@ If you are using Docker Desktop for Windows then run the following commands: ```bash git checkout 3.0.0 set TAG=3.0.0 -docker-compose -f docker-compose-non-dev.yml pull -docker-compose -f docker-compose-non-dev.yml up +docker compose -f docker-compose-non-dev.yml pull +docker compose -f docker-compose-non-dev.yml up ``` :::tip @@ -97,14 +97,14 @@ Note that some configuration is mandatory for production instances of Superset. ::: :::caution All of the content belonging to a Superset instance - charts, dashboards, users, etc. - is stored in its metadata database. In production, this database should be backed up. -The default installation with docker-compose will store that data in a PostgreSQL database contained in a Docker [volume](https://docs.docker.com/storage/volumes/), +The default installation with docker compose will store that data in a PostgreSQL database contained in a Docker [volume](https://docs.docker.com/storage/volumes/), which is not backed up. To avoid risking data loss, either use a managed database for your metadata (recommended) or perform your own regular backups by extracting and storing the contents of the default PostgreSQL database from its volume (here's an [example of how to dump and restore](https://stackoverflow.com/questions/24718706/backup-restore-a-dockerized-postgresql-database)). ::: You should see a wall of logging output from the containers being launched on your machine. Once this output slows, you should have a running instance of Superset on your local machine! To -avoid the wall of text on future runs, add the `-d` option to the end of the `docker-compose up` command. +avoid the wall of text on future runs, add the `-d` option to the end of the `docker compose up` command. #### Configuring Docker Compose @@ -124,7 +124,7 @@ Users often want to connect to other databases from Superset. Currently, the eas :::note Superset uses [Scarf Gateway](https://about.scarf.sh/scarf-gateway) to collect telemetry data. Knowing the installation counts for different Superset versions informs the project's decisions about patching and long-term support. Scarf purges personally identifiable information (PII) and provides only aggregated statistics. -To opt-out of this data collection in your docker-compose based installation, edit the `x-superset-image:` line in your `docker-compose.yml` and `docker-compose-non-dev.yml` files, replacing `apachesuperset.docker.scarf.sh/apache/superset` with `apache/superset` to pull the image directly from Docker Hub. +To opt-out of this data collection in your docker compose based installation, edit the `x-superset-image:` line in your `docker-compose.yml` and `docker-compose-non-dev.yml` files, replacing `apachesuperset.docker.scarf.sh/apache/superset` with `apache/superset` to pull the image directly from Docker Hub. ::: ### 4. Log in to Superset @@ -146,7 +146,7 @@ password: admin ### 5. Connecting Superset to your local database instance -When running Superset using `docker` or `docker-compose` it runs in its own docker container, as if the Superset was running in a separate machine entirely. Therefore attempts to connect to your local database with the hostname `localhost` won't work as `localhost` refers to the docker container Superset is running in, and not your actual host machine. Fortunately, docker provides an easy way to access network resources in the host machine from inside a container, and we will leverage this capability to connect to our local database instance. +When running Superset using `docker` or `docker compose` it runs in its own docker container, as if the Superset was running in a separate machine entirely. Therefore attempts to connect to your local database with the hostname `localhost` won't work as `localhost` refers to the docker container Superset is running in, and not your actual host machine. Fortunately, docker provides an easy way to access network resources in the host machine from inside a container, and we will leverage this capability to connect to our local database instance. Here the instructions are for connecting to postgresql (which is running on your host machine) from Superset (which is running in its docker container). Other databases may have slightly different configurations but gist would be same and boils down to 2 steps - diff --git a/docs/docs/installation/upgrading-superset.mdx b/docs/docs/installation/upgrading-superset.mdx index c1a810e4969dd..167c5acae50ec 100644 --- a/docs/docs/installation/upgrading-superset.mdx +++ b/docs/docs/installation/upgrading-superset.mdx @@ -12,7 +12,7 @@ version: 1 First make sure to wind down the running containers in Docker Compose: ```bash -docker-compose down +docker compose down ``` Then, update the folder that mirrors the `superset` repo through git: @@ -25,7 +25,7 @@ git pull origin master Then, restart the containers and any changed Docker images will be automatically pulled down: ```bash -docker-compose up +docker compose up ``` ### Updating Superset Manually diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh index 2c3b5bf359733..88fa41d44bd94 100755 --- a/scripts/tests/run.sh +++ b/scripts/tests/run.sh @@ -26,7 +26,7 @@ function reset_db() { echo -------------------- echo Resetting test DB echo -------------------- - docker-compose stop superset-tests-worker superset || true + docker compose stop superset-tests-worker superset || true RESET_DB_CMD="psql \"postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432\" <<-EOF DROP DATABASE IF EXISTS ${DB_NAME}; CREATE DATABASE ${DB_NAME}; @@ -38,7 +38,7 @@ function reset_db() { EOF " docker exec -i superset_db bash -c "${RESET_DB_CMD}" - docker-compose start superset-tests-worker superset + docker compose start superset-tests-worker superset } # diff --git a/tox.ini b/tox.ini index c63fa947df6f4..be79ff4cf8dbe 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ setenv = # docker run -p 8080:8080 --name presto starburstdata/presto mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default # based on https://github.com/big-data-europe/docker-hadoop - # clone the repo & run docker-compose up -d to test locally + # clone the repo & run docker compose up -d to test locally mysql-hive: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 mysql-hive: SUPERSET__SQLALCHEMY_EXAMPLES_URI = hive://localhost:10000/default # make sure that directory is accessible by docker