From 14ba6d6bd1d6e561823a4dea5a2958dfa717386b Mon Sep 17 00:00:00 2001 From: Scott-Simmons <52365471+Scott-Simmons@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:38:09 +1100 Subject: [PATCH 1/5] Add a dockerfile for testing Add image for testing tsfresh with multiple python versions. This can be a good final sanity check e.g. before a release. --- Dockerfile.testing | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Dockerfile.testing diff --git a/Dockerfile.testing b/Dockerfile.testing new file mode 100644 index 00000000..4641ea3c --- /dev/null +++ b/Dockerfile.testing @@ -0,0 +1,62 @@ +# Bakes the python versions which tsfresh targets into a testing env +FROM ubuntu:22.04 + +SHELL ["/bin/bash", "-c"] + +# These are required to build python from source +RUN apt-get update && apt-get install -y \ + python3-pip \ + curl \ + clang \ + git \ + build-essential \ + libssl-dev \ + libreadline-dev \ + zlib1g-dev \ + libbz2-dev \ + libsqlite3-dev \ + llvm \ + libncurses5-dev \ + libgdbm-dev \ + libnss3-dev \ + libffi-dev \ + liblzma-dev \ + libgmp-dev \ + libmpfr-dev \ + && apt-get clean + + +RUN curl https://pyenv.run | bash + +# For interactive use (if any), this is an edge case. +RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \ + echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \ + echo 'eval "$(pyenv init -)"' >> ~/.bashrc + +ENV PYENV_ROOT="/root/.pyenv" +ENV PATH="$PYENV_ROOT/bin:$PATH" +ENV PATH="$PYENV_ROOT/shims:$PATH" + +ARG PYTHON_VERSIONS +RUN for version in $PYTHON_VERSIONS; do \ + echo Installing $version; \ + # Band aid for https://github.com/pyenv/pyenv/issues/1738 + # since this also appears to apply to 3.7.X + if [[ $version =~ ^3\.7\..*$ ]]; then \ + echo Using clang to compile $version; \ + CC=clang pyenv install $version || exit 1; \ + else \ + pyenv install $version || exit 1; \ + fi; \ + done + +RUN pyenv global $PYTHON_VERSIONS + +RUN pip install tox + +WORKDIR /tsfresh + +# Requires adding safe.directory so that tsfresh can build when the +# repo is mounted. +# Note cannot do this at build time as no git directory exists +CMD ["/bin/bash", "-c", "git config --global --add safe.directory /tsfresh && tox -r -p auto"] From b7203980d1986b3abf067c82062294e140c225cc Mon Sep 17 00:00:00 2001 From: Scott-Simmons <52365471+Scott-Simmons@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:40:28 +1100 Subject: [PATCH 2/5] Nit: add document start on yaml file --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d3b6089..d0754db1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +--- repos: - repo: https://github.com/psf/black rev: 22.12.0 From aed52b78a0c964792a40df1254b4db056c959942 Mon Sep 17 00:00:00 2001 From: Scott-Simmons <52365471+Scott-Simmons@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:42:02 +1100 Subject: [PATCH 3/5] Initialise makefile Targets include 1. Building dockerised testing environment 2. Testing tsfresh within testing environment 3. Testing tsfresh locally with tox with available python versions 4. Testing tsfresh locally with current active python version 5. Cleaning build artifacts 6. Formatting. This is so that changes can be made before pushing the changes up the remote to trigger the formatting checks. TODO: pre-commit hooks should probs be added too. --- Makefile | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..0909f3e5 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +WORKDIR := /tsfresh +TEST_IMAGE := tsfresh-test-image +TEST_DOCKERFILE := Dockerfile.testing +TEST_CONTAINER := tsfresh-test-container +# >= 3.9.2 ---> https://github.com/dask/distributed/issues/7956 +PYTHON_VERSIONS := "3.7.12 3.8.12 3.9.12 3.10.12 3.11.0" +BLACK_VERSION := 22.12.0 +# Isort 5.12.0 not supported with python 3.7.12 +ISORT_VERSION := 5.12.0 + +build-testenv: + docker build \ + -f $(TEST_DOCKERFILE) \ + -t $(TEST_IMAGE) \ + --build-arg PYTHON_VERSIONS=$(PYTHON_VERSIONS) \ + . + +# Tests `PYTHON_VERSIONS`, provided they are also +# specified in setup.cfg `envlist` +test-all-testenv: clean build-testenv + docker run --rm \ + --name $(TEST_CONTAINER) \ + -v .:$(WORKDIR) \ + -v build_artifacts:$(WORKDIR)/build \ + -v tox_artifacts:$(WORKDIR)/.tox \ + -v egg_artifacts:$(WORKDIR)/tsfresh.egg-info \ + $(TEST_IMAGE) + +# Tests the python binaries installed +# on local machine, provided they are also +# specified in setup.cfg `envlist` +test-all-local: clean + tox -r -p auto + +# Tests for python version on local machine in +# current context (e.g. global or local version of +# python set by pyenv, or the python version in +# the active virtualenv). +test-local: clean + pip install .[testing] + pytest + +clean: + rm -rf .tox build/ dist/ *.egg-info + +install-formatters: + pip install black==$(BLACK_VERSION) isort==$(ISORT_VERSION) + +format: install-formatters + black --extend-exclude \.docs . + isort --profile black --skip-glob="docs" . + +.PHONY: clean test-all-local test-local test-all-testenv format install-formatters From 27f02b91eb244d59d897f0a3ae5362cfe8afe2b9 Mon Sep 17 00:00:00 2001 From: Scott-Simmons <52365471+Scott-Simmons@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:47:57 +1100 Subject: [PATCH 4/5] Documentation updates Change docs to reflect the new testing invokations. --- docs/text/how_to_contribute.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/text/how_to_contribute.rst b/docs/text/how_to_contribute.rst index 0633784f..a5108eec 100644 --- a/docs/text/how_to_contribute.rst +++ b/docs/text/how_to_contribute.rst @@ -64,7 +64,7 @@ you have to: .. code:: - pytest + make test-local To test changes across multiple versions of Python, run: @@ -72,13 +72,24 @@ To test changes across multiple versions of Python, run: .. code:: - tox -r + make test-all-local -`tox -r` will execute tests for the Python versions specified in `setup.cfg `_ using the `envlist` variable. For example, if `envlist` is set to `py37, py38`, the test suite will run for Python 3.7 and 3.8 on the local development platform, assuming the binaries for those versions are available locally. The exact Python microversions (e.g. `3.7.1` vs `3.7.2`) depend on what is installed on the local development machine. +This will execute tests for the Python versions specified in `setup.cfg `_ using the `envlist` variable. For example, if `envlist` is set to `py37, py38`, the test suite will run for Python 3.7 and 3.8 on the local development platform, assuming the binaries for those versions are available locally. The exact Python microversions (e.g. `3.7.1` vs `3.7.2`) depend on what is installed on the local development machine. A recommended way to manage multiple Python versions when testing locally is with `pyenv`, which enables organized installation and switching between versions. +In addition to running tests locally, you can also run them in a Dockerized testing environment: + + +.. code:: + + make test-all-testenv + + +This command will initially take some time. However subsequent invokations will be faster, and testing this way ensures a clean, consistent test environment, regardless of your local setup. + + Documentation ''''''''''''' From 3ddad5985542de57c11229ccc6a1f47a1e49ec04 Mon Sep 17 00:00:00 2001 From: Scott-Simmons <52365471+Scott-Simmons@users.noreply.github.com> Date: Mon, 18 Nov 2024 23:31:04 +1100 Subject: [PATCH 5/5] Responded to PR review - Remove the python PATCH version from makefile - Move calls to `make clean` to be part of the docker test invokation instead of within local test commands - Removed the one line makefile commands (the docs have enough info on the development process so will not bother using `make` for these. - Updated hyperlink to point to main instead of specific commit sha --- Makefile | 38 +++++++-------------------------- docs/text/how_to_contribute.rst | 6 +++--- 2 files changed, 11 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 0909f3e5..ce6c3f55 100644 --- a/Makefile +++ b/Makefile @@ -2,22 +2,20 @@ WORKDIR := /tsfresh TEST_IMAGE := tsfresh-test-image TEST_DOCKERFILE := Dockerfile.testing TEST_CONTAINER := tsfresh-test-container -# >= 3.9.2 ---> https://github.com/dask/distributed/issues/7956 -PYTHON_VERSIONS := "3.7.12 3.8.12 3.9.12 3.10.12 3.11.0" -BLACK_VERSION := 22.12.0 -# Isort 5.12.0 not supported with python 3.7.12 -ISORT_VERSION := 5.12.0 +PYTHON_VERSIONS := "3.7 3.8 3.9 3.10 3.11" -build-testenv: +# Tests `PYTHON_VERSIONS`, provided they are also +# specified in setup.cfg `envlist` +test-all-testenv: build-docker-testenv run-docker-tests clean + +build-docker-testenv: docker build \ -f $(TEST_DOCKERFILE) \ -t $(TEST_IMAGE) \ --build-arg PYTHON_VERSIONS=$(PYTHON_VERSIONS) \ . -# Tests `PYTHON_VERSIONS`, provided they are also -# specified in setup.cfg `envlist` -test-all-testenv: clean build-testenv +run-docker-tests: docker run --rm \ --name $(TEST_CONTAINER) \ -v .:$(WORKDIR) \ @@ -26,28 +24,8 @@ test-all-testenv: clean build-testenv -v egg_artifacts:$(WORKDIR)/tsfresh.egg-info \ $(TEST_IMAGE) -# Tests the python binaries installed -# on local machine, provided they are also -# specified in setup.cfg `envlist` -test-all-local: clean - tox -r -p auto - -# Tests for python version on local machine in -# current context (e.g. global or local version of -# python set by pyenv, or the python version in -# the active virtualenv). -test-local: clean - pip install .[testing] - pytest - clean: rm -rf .tox build/ dist/ *.egg-info -install-formatters: - pip install black==$(BLACK_VERSION) isort==$(ISORT_VERSION) - -format: install-formatters - black --extend-exclude \.docs . - isort --profile black --skip-glob="docs" . -.PHONY: clean test-all-local test-local test-all-testenv format install-formatters +.PHONY: build-docker-testenv clean run-docker-tests test-all-testenv diff --git a/docs/text/how_to_contribute.rst b/docs/text/how_to_contribute.rst index a5108eec..4f169e44 100644 --- a/docs/text/how_to_contribute.rst +++ b/docs/text/how_to_contribute.rst @@ -64,7 +64,7 @@ you have to: .. code:: - make test-local + pytest To test changes across multiple versions of Python, run: @@ -72,10 +72,10 @@ To test changes across multiple versions of Python, run: .. code:: - make test-all-local + tox -r -p auto -This will execute tests for the Python versions specified in `setup.cfg `_ using the `envlist` variable. For example, if `envlist` is set to `py37, py38`, the test suite will run for Python 3.7 and 3.8 on the local development platform, assuming the binaries for those versions are available locally. The exact Python microversions (e.g. `3.7.1` vs `3.7.2`) depend on what is installed on the local development machine. +This will execute tests for the Python versions specified in `setup.cfg `_ using the `envlist` variable. For example, if `envlist` is set to `py37, py38`, the test suite will run for Python 3.7 and 3.8 on the local development platform, assuming the binaries for those versions are available locally. The exact Python microversions (e.g. `3.7.1` vs `3.7.2`) depend on what is installed on the local development machine. A recommended way to manage multiple Python versions when testing locally is with `pyenv`, which enables organized installation and switching between versions.