Skip to content
This repository has been archived by the owner on Nov 30, 2023. It is now read-only.

Commit

Permalink
Fix installation of common Python packages (#1424)
Browse files Browse the repository at this point in the history
* Fix package installation

* Update comment

* Move python to front of PATH

* Add PYTHON parameter

* Move ML packages to new script

* Remove invalid docs

* Check for PYTHON

* Install packages from argument

* Resolve comments

* Add tests for new packages

* Copy JupyterLab script

* Use generic package install script

* Add new script to README

* Add missing $

* Try to fix smoke test

* Revert changes to jupyterlab-debian.sh

* Fix directory of Python

* Remove busted sudos

* Update test.sh

Co-authored-by: Josh Spicer <[email protected]>
  • Loading branch information
jungaretti and joshspicer authored May 4, 2022
1 parent ab35cd6 commit f7b92c8
Show file tree
Hide file tree
Showing 6 changed files with 179 additions and 5 deletions.
19 changes: 15 additions & 4 deletions containers/codespaces-linux/.devcontainer/base.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ ENV SHELL=/bin/bash \
JUPYTERLAB_PATH="${HOMEDIR}/.local/bin" \
DOCKER_BUILDKIT=1

ENV PATH="${NVM_DIR}/current/bin:${NPM_GLOBAL}/bin:${ORIGINAL_PATH}:${DOTNET_ROOT}:${DOTNET_ROOT}/tools:${SDKMAN_DIR}/bin:${SDKMAN_DIR}/candidates/gradle/current/bin:${SDKMAN_DIR}/candidates/java/current/bin:/opt/maven/lts:${GOROOT}/bin:${GOPATH}/bin:${PIPX_BIN_DIR}:/opt/conda/condabin:${JAVA_ROOT}/current/bin:${NODE_ROOT}/current/bin:${PHP_ROOT}/current/bin:${PYTHON_ROOT}/current/bin:${RUBY_ROOT}/current/bin:${MAVEN_ROOT}/current/bin:${HUGO_ROOT}/current/bin:${JUPYTERLAB_PATH}:${ORYX_PATHS}"
ENV PATH="${NVM_DIR}/current/bin:${NPM_GLOBAL}/bin:${PYTHON_ROOT}/current/bin:${ORIGINAL_PATH}:${DOTNET_ROOT}:${DOTNET_ROOT}/tools:${SDKMAN_DIR}/bin:${SDKMAN_DIR}/candidates/gradle/current/bin:${SDKMAN_DIR}/candidates/java/current/bin:/opt/maven/lts:${GOROOT}/bin:${GOPATH}/bin:${PIPX_BIN_DIR}:/opt/conda/condabin:${JAVA_ROOT}/current/bin:${NODE_ROOT}/current/bin:${PHP_ROOT}/current/bin:${RUBY_ROOT}/current/bin:${MAVEN_ROOT}/current/bin:${HUGO_ROOT}/current/bin:${JUPYTERLAB_PATH}:${ORYX_PATHS}"

# Install needed utilities and setup non-root user. Use a separate RUN statement to add your own dependencies.
COPY library-scripts/* setup-user.sh first-run-notice.txt /tmp/scripts/
Expand Down Expand Up @@ -68,10 +68,21 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
&& mkdir -p /usr/local/etc/vscode-dev-containers/ \
&& mv -f /tmp/scripts/first-run-notice.txt /usr/local/etc/vscode-dev-containers/

# Install Python, PHP, Ruby utilities, and JupyterLab
# Install Python, JupyterLab, common machine learning packages, and Ruby utilities
RUN bash /tmp/scripts/python-debian.sh "none" "/opt/python/latest" "${PIPX_HOME}" "${USERNAME}" "true" \
&& bash /tmp/scripts/jupyterlab-debian.sh \
&& pip install --user numpy pandas scipy matplotlib seaborn scikit-learn tensorflow keras torch requests \
# Install JupyterLab and common machine learning packages
&& PYTHON_BINARY="${PYTHON_ROOT}/current/bin/python" \
&& bash /tmp/scripts/python-package-debian.sh "jupyterlab" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "numpy" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "pandas" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "scipy" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "matplotlib" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "seaborn" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "scikit-learn" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "tensorflow" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "keras" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "torch" "latest" $PYTHON_BINARY \
&& bash /tmp/scripts/python-package-debian.sh "requests" "latest" $PYTHON_BINARY \
# Install rvm, rbenv, any missing base gems
&& chown -R ${USERNAME} /opt/ruby/* \
&& bash /tmp/scripts/ruby-debian.sh "none" "${USERNAME}" "true" "true" \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
#-------------------------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
#-------------------------------------------------------------------------------------------------------------
#
# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/python-package-debian.md
# Maintainer: The VS Code and Codespaces Teams
#
# Syntax: ./python-package-debian.sh

set -e

PACKAGE=${1:-""}
VERSION=${2:-"latest"}
PYTHON=${3:-"python"}
USERNAME=${4-"automatic"}

# If in automatic mode, determine if a user already exists, if not use vscode
if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then
USERNAME=""
POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)")
for CURRENT_USER in ${POSSIBLE_USERS[@]}; do
if id -u ${CURRENT_USER} > /dev/null 2>&1; then
USERNAME=${CURRENT_USER}
break
fi
done
if [ "${USERNAME}" = "" ]; then
USERNAME=vscode
fi
elif [ "${USERNAME}" = "none" ]; then
USERNAME=root
USER_UID=0
USER_GID=0
fi

# Use sudo to run as non-root user is not already running
sudoUserIf()
{
if [ "$(id -u)" -eq 0 ] && [ "${USERNAME}" != "root" ]; then
sudo -u ${USERNAME} "$@"
else
"$@"
fi
}

# Make sure that Python is installed correctly
if ! ${PYTHON} --version > /dev/null ; then
echo "Python command not found: ${PYTHON}"
exit 1
fi

# pip skips installation when a requirement is already satisfied
if [ ${VERSION} = "latest" ]; then
sudoUserIf ${PYTHON} -m pip install ${PACKAGE} --no-cache-dir
else
sudoUserIf ${PYTHON} -m pip install ${PACKAGE}==${VERSION} --no-cache-dir
fi
16 changes: 15 additions & 1 deletion containers/codespaces-linux/test-project/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ check "dotnet-6-installed-by-oryx" dotnet --info | grep "/opt/dotnet/6\.0\.[0-9]

# Check Python
check "python" python --version
check "pip" pip3 --version
check "python3" python3 --version
check "pip" pip --version
check "pip3" pip3 --version
check "pipx" pipx --version
check "pylint" pylint --version
check "flake8" flake8 --version
Expand All @@ -31,6 +33,18 @@ check "pydocstyle" pydocstyle --version
check "bandit" bandit --version
check "virtualenv" virtualenv --version

# # Check Python packages
# check "numpy" python -c 'import numpy'
# check "pandas" python -c 'import pandas'
# check "scipy" python -c 'import scipy'
# check "matplotlib" python -c 'import matplotlib'
# check "seaborn" python -c 'import seaborn'
# check "scikit-learn" python -c 'import sklearn'
# check "tensorflow" python -c 'import tensorflow'
# check "keras" python -c 'import keras'
# check "torch" python -c 'import torch'
# check "requests" python -c 'import requests'

# Check JupyterLab
check "jupyter-lab" jupyter-lab --version

Expand Down
1 change: 1 addition & 0 deletions script-library/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ Some scripts have special installation instructions (like `desktop-lite-debian.s
| [Node.js Install Script](docs/node.md) | `node-debian.sh` | VS Code and GitHub Codespaces teams|
| [PowerShell Install Script](docs/powershell.md) | `powershell-debian.sh` | VS Code and GitHub Codespaces teams|
| [Python Install Script](docs/python.md) | `python-debian.sh` | VS Code and GitHub Codespaces teams|
| [Python Package Install Script](docs/python-package.md) | `python-package-debian.sh` | VS Code and GitHub Codespaces teams|
| [Ruby Install Script](docs/ruby.md) | `ruby-debian.sh` | VS Code and GitHub Codespaces teams|
| [Rust (rustlang) Install Script](docs/rust.md) | `rust-debian.sh` | VS Code and GitHub Codespaces teams|
| [SSH Server Install Script](docs/sshd.md) | `sshd-debian.sh` | VS Code and GitHub Codespaces teams|
Expand Down
30 changes: 30 additions & 0 deletions script-library/docs/python-package.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Python Package Install Script

*Installs a Python package.*

**Script status**: Stable

**OS support**: Debian 9+, Ubuntu 18.04+, and downstream distros.

**Maintainer**: GitHub Codespaces team

## Syntax

```text
./python-package-debian.sh PACKAGE [VERSION] [PYTHON_BINARY] [USERNAME]
```

## Usage

### Script use

1. Add [`python-package-debian.sh`](../python-package-debian.sh) to `.devcontainer/library-scripts`

2. Add the following to your `.devcontainer/Dockerfile`:

```Dockerfile
COPY library-scripts/python-package-debian.sh /tmp/library-scripts/
RUN bash /tmp/library-scripts/python-package-debian.sh PACKAGE
```

That's it!
59 changes: 59 additions & 0 deletions script-library/python-package-debian.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
#-------------------------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
#-------------------------------------------------------------------------------------------------------------
#
# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/python-package-debian.md
# Maintainer: The VS Code and Codespaces Teams
#
# Syntax: ./python-package-debian.sh

set -e

PACKAGE=${1:-""}
VERSION=${2:-"latest"}
PYTHON=${3:-"python"}
USERNAME=${4-"automatic"}

# If in automatic mode, determine if a user already exists, if not use vscode
if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then
USERNAME=""
POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)")
for CURRENT_USER in ${POSSIBLE_USERS[@]}; do
if id -u ${CURRENT_USER} > /dev/null 2>&1; then
USERNAME=${CURRENT_USER}
break
fi
done
if [ "${USERNAME}" = "" ]; then
USERNAME=vscode
fi
elif [ "${USERNAME}" = "none" ]; then
USERNAME=root
USER_UID=0
USER_GID=0
fi

# Use sudo to run as non-root user is not already running
sudoUserIf()
{
if [ "$(id -u)" -eq 0 ] && [ "${USERNAME}" != "root" ]; then
sudo -u ${USERNAME} "$@"
else
"$@"
fi
}

# Make sure that Python is installed correctly
if ! ${PYTHON} --version > /dev/null ; then
echo "Python command not found: ${PYTHON}"
exit 1
fi

# pip skips installation when a requirement is already satisfied
if [ ${VERSION} = "latest" ]; then
sudoUserIf ${PYTHON} -m pip install ${PACKAGE} --no-cache-dir
else
sudoUserIf ${PYTHON} -m pip install ${PACKAGE}==${VERSION} --no-cache-dir
fi

0 comments on commit f7b92c8

Please sign in to comment.