From 9fe94115a510a9fc29b286442a0ce852b7112504 Mon Sep 17 00:00:00 2001 From: Daniel Bruno Date: Sun, 16 Apr 2023 06:30:06 -0400 Subject: [PATCH 1/3] Added support for EMR 6.10.0 and pytest --- templates/devcontainer.json | 2 +- templates/pyspark.dockerfile | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/devcontainer.json b/templates/devcontainer.json index d358a09..e881a7c 100644 --- a/templates/devcontainer.json +++ b/templates/devcontainer.json @@ -7,7 +7,7 @@ // Update 'VARIANT' to pick an Debian version: bullseye, buster // Use bullseye on local arm64/Apple Silicon. "args": { - "RELEASE": "emr-6.6.0", + "RELEASE": "emr-6.10.0", "RELEASE_TAG": "latest", "REGION": "us-west-2", "EMR_ACCOUNT_ID": "895885662937" diff --git a/templates/pyspark.dockerfile b/templates/pyspark.dockerfile index 165d208..b152f6b 100644 --- a/templates/pyspark.dockerfile +++ b/templates/pyspark.dockerfile @@ -1,6 +1,6 @@ # See here for image details: https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/docker-custom-images-steps.html # Arguments used to build the image URI - update to your desired region/EMR release version per the link above -ARG RELEASE="emr-6.6.0" +ARG RELEASE="emr-6.10.0" ARG RELEASE_TAG="latest" ARG REGION="us-west-2" ARG EMR_ACCOUNT_ID="895885662937" @@ -45,5 +45,8 @@ RUN python3 -m pip install -U pip # Enable Jupyter notebooks RUN python3 -m pip install ipykernel +# Enable pytest +run python3 -m pip install pytest + # Switch back to the default user USER hadoop:hadoop From 93bdcaeb2b8e30e807fa26a4539792f7846cf57f Mon Sep 17 00:00:00 2001 From: Daniel Bruno Date: Sun, 16 Apr 2023 09:52:11 -0400 Subject: [PATCH 2/3] fixed formatting --- templates/pyspark.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/pyspark.dockerfile b/templates/pyspark.dockerfile index b152f6b..27c7927 100644 --- a/templates/pyspark.dockerfile +++ b/templates/pyspark.dockerfile @@ -46,7 +46,7 @@ RUN python3 -m pip install -U pip RUN python3 -m pip install ipykernel # Enable pytest -run python3 -m pip install pytest +RUN python3 -m pip install pytest # Switch back to the default user USER hadoop:hadoop From 185e9f2d139f56f49ff5c2dde44e0c8204b2c1b9 Mon Sep 17 00:00:00 2001 From: Daniel Bruno <106827135+dabrun@users.noreply.github.com> Date: Tue, 25 Apr 2023 14:09:06 -0400 Subject: [PATCH 3/3] Update templates/pyspark.dockerfile collapsed into one pip install Co-authored-by: Damon P. Cortesi --- templates/pyspark.dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/pyspark.dockerfile b/templates/pyspark.dockerfile index 27c7927..062b45b 100644 --- a/templates/pyspark.dockerfile +++ b/templates/pyspark.dockerfile @@ -42,11 +42,10 @@ RUN if [ "$TARGETARCH" != "amd64" ]; then yum install -y gcc python3-devel; fi # Upgrade pip first RUN python3 -m pip install -U pip -# Enable Jupyter notebooks -RUN python3 -m pip install ipykernel - -# Enable pytest -RUN python3 -m pip install pytest +# Enable Jupyter notebooks and pytest +RUN python3 -m pip install \ + ipykernel \ + pytest # Switch back to the default user USER hadoop:hadoop