diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index f1547bb4..8482f6db 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -3,9 +3,6 @@ services: dynamodb: command: "-jar DynamoDBLocal.jar -sharedDb -inMemory" image: "amazon/dynamodb-local:latest" - container_name: dynamodb - networks: - - scylla expose: - 8001 ports: @@ -14,8 +11,6 @@ services: cassandra: image: cassandra:latest - networks: - - scylla volumes: - ./tests/docker/cassandra:/var/lib/cassandra ports: @@ -25,8 +20,6 @@ services: scylla-source: image: scylladb/scylla:latest - networks: - - scylla volumes: - "./tests/docker/scylla-source:/var/lib/scylla" ports: @@ -38,8 +31,6 @@ services: scylla: image: scylladb/scylla:latest - networks: - - scylla volumes: - "./tests/docker/scylla:/var/lib/scylla" ports: @@ -56,33 +47,17 @@ services: volumes: - "./tests/docker/s3:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" - networks: - - scylla spark-master: build: dockerfiles/spark command: master - container_name: spark-master - hostname: spark-master environment: - SPARK_PUBLIC_DNS: spark-master - networks: - - scylla + SPARK_PUBLIC_DNS: localhost expose: - 5005 - - 7001 - - 7002 - - 7003 - - 7004 - - 7005 - - 7006 - - 7077 - - 6066 ports: - 4040:4040 - 5005:5005 - - 6066:6066 - - 7077:7077 - 8080:8080 volumes: - ./migrator/target/scala-2.13:/jars @@ -93,22 +68,13 @@ services: spark-worker: build: dockerfiles/spark command: worker - hostname: spark-worker - container_name: spark-worker-1 environment: SPARK_WORKER_CORES: 3 SPARK_WORKER_MEMORY: 1024m SPARK_WORKER_WEBUI_PORT: 8081 - SPARK_PUBLIC_DNS: spark-worker - networks: - - scylla + SPARK_PUBLIC_DNS: localhost expose: - 5006 - - 7012 - - 7013 - - 7014 - - 7015 - - 7016 ports: - 5006:5006 - 8081:8081 @@ -116,6 +82,3 @@ services: - ./tests/docker/parquet:/app/parquet depends_on: - spark-master - -networks: - scylla: diff --git a/docker-compose.yaml b/docker-compose.yaml index 3ac93386..61f45cf0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,25 +2,10 @@ services: spark-master: build: dockerfiles/spark command: master - container_name: spark-master - hostname: spark-master environment: - SPARK_PUBLIC_DNS: spark-master - networks: - - spark - expose: - - 7001 - - 7002 - - 7003 - - 7004 - - 7005 - - 7006 - - 7077 - - 6066 + SPARK_PUBLIC_DNS: localhost ports: - 4040:4040 - - 6066:6066 - - 7077:7077 - 8080:8080 volumes: - ./data/spark-master:/tmp/data @@ -31,22 +16,11 @@ services: spark-worker: build: dockerfiles/spark command: worker - hostname: spark-worker - container_name: spark-worker-1 environment: SPARK_WORKER_CORES: 3 SPARK_WORKER_MEMORY: 1024m SPARK_WORKER_WEBUI_PORT: 8081 - SPARK_PUBLIC_DNS: spark-worker - networks: - - spark - expose: - - 7012 - - 7013 - - 7014 - - 7015 - - 7016 - - 8881 + SPARK_PUBLIC_DNS: localhost ports: - 8081:8081 volumes: @@ -54,6 +28,3 @@ services: - ./parquet-data:/parquet-data depends_on: - spark-master - -networks: - spark: diff --git a/docs/source/getting-started/docker.rst b/docs/source/getting-started/docker.rst index 6d4653cd..d44e8053 100644 --- a/docs/source/getting-started/docker.rst +++ b/docs/source/getting-started/docker.rst @@ -31,13 +31,6 @@ This page describes how to set up a Spark cluster locally on your machine by usi http://localhost:8080 - Tip: add the following aliases to your ``/etc/hosts`` to make links work in the Spark UI - - .. code-block:: text - - 127.0.0.1 spark-master - 127.0.0.1 spark-worker - 5. Rename the file ``config.yaml.example`` to ``config.yaml``, and `configure <../#configure-the-migration>`_ it according to your needs. 6. Finally, run the migration.