From 40a30972fec74830068dc5cc406696cd0059bd35 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 4 Jun 2024 13:23:52 +0800 Subject: [PATCH 01/14] remove pytest-snail from dev requirements file fixes #2908 --- requirements/dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index a99a52bd99..777337ae83 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,7 +12,6 @@ packaging pre-commit pytest pytest-benchmark -pytest-snail read-version setuptools towncrier From f0d94494b46678e630c4dc3f454efff981625de3 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 4 Jun 2024 15:46:09 +0800 Subject: [PATCH 02/14] increased windows no-output timeout to 20m --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ee8f9c0df..b6355014cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -167,7 +167,7 @@ jobs: py_version: << parameters.py_version >> - run: name: Testing Hydra - no_output_timeout: 10m + no_output_timeout: 20m command: | $env:NOX_PYTHON_VERSIONS="<< parameters.py_version >>" $env:ConEmuDefaultCp=65001 From 503f6dfbe673c3b5b38da4be0db11a605f4c8e91 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 4 Jun 2024 17:53:33 +0800 Subject: [PATCH 03/14] fixed plugins bandit warnings --- .../integration_test_tools/create_integration_test_ami.py | 2 +- .../integration_test_tools/setup_integration_test_ami.py | 2 +- plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py b/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py index dc2ab834e7..1f5bdc67a7 100644 --- a/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py +++ b/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py @@ -24,7 +24,7 @@ def _run_command(command: str) -> str: print(f"{str(datetime.now())} - Running: {command}") - output = subprocess.getoutput(command) + output = subprocess.getoutput(command) # nosec B605 print(f"{str(datetime.now())} - {output}") return output diff --git a/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py b/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py index f114e67b42..a49fccd692 100644 --- a/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py +++ b/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py @@ -16,7 +16,7 @@ def _run_command(command: str) -> str: print(f"{str( datetime.now() )} - OUT: {command}") - output = subprocess.getoutput(command) + output = subprocess.getoutput(command) # nosec B605 print(f"{str( datetime.now() )} - OUT: {output}") return output diff --git a/plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py b/plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py index c06aa2c67e..82cb1f0608 100644 --- a/plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py +++ b/plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py @@ -129,7 +129,7 @@ def run_command(commands: str) -> str: log.info(f"running: {commands}") - output = subprocess.getoutput(commands) + output = subprocess.getoutput(commands) # nosec B605 log.info(f"outputs: {output}") return output From 05bddbf006751c5c850dae73e4beba8d04b3ced1 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 4 Jun 2024 17:53:43 +0800 Subject: [PATCH 04/14] minor changes to noxfile and circle config --- .circleci/config.yml | 3 ++- noxfile.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b6355014cf..5cf84a36cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,7 @@ commands: key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ .Branch }}-<< parameters.py_version >> - run: name: Preparing environment - system + no_output_timeout: 20m command: | choco install -y --no-progress miniconda3 choco install -y --no-progress openssl openjdk11jre @@ -167,7 +168,7 @@ jobs: py_version: << parameters.py_version >> - run: name: Testing Hydra - no_output_timeout: 20m + no_output_timeout: 10m command: | $env:NOX_PYTHON_VERSIONS="<< parameters.py_version >>" $env:ConEmuDefaultCp=65001 diff --git a/noxfile.py b/noxfile.py index 30a27a1f1c..442356b1c0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -87,13 +87,13 @@ def print_installed_package_version(session: Session, package_name: str) -> None pip_list: str = session.run("pip", "list", silent=True) for line in pip_list.split("\n"): if package_name in line: - print(f"Installed {package_name} version: {line}") + logger.info(f"Installed {package_name} version: {line}") def install_hydra(session: Session, cmd: List[str]) -> None: # needed for build session.install("read-version", silent=SILENT) - # clean install hydra + # clean-install hydra session.chdir(BASE) if USE_OMEGACONF_DEV_VERSION: session.install("--pre", "omegaconf", silent=SILENT) From f82ecbcaad26baddd2410cdfe24ce769d7b368e8 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 4 Jun 2024 19:58:40 +0800 Subject: [PATCH 05/14] upgrading openjdk in Windows and removing openssl --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5cf84a36cc..e28555e921 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,7 +101,7 @@ commands: no_output_timeout: 20m command: | choco install -y --no-progress miniconda3 - choco install -y --no-progress openssl openjdk11jre + choco install -y --no-progress openjdk C:\tools\miniconda3\Scripts\conda.exe init powershell - run: name: Preparing environment - Hydra From bb26db78adaab8be3673e82b9a3ba1c95acca98c Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Fri, 7 Jun 2024 21:53:28 +0800 Subject: [PATCH 06/14] Fixing macos by untapping broken brew casks --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e28555e921..1a574c10c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,9 @@ commands: - run: name: Preparing environment - Brew command: | + brew --version + brew untap Homebrew/homebrew-cask-versions + brew untap adoptopenjdk/openjdk brew update brew upgrade git brew install fish @@ -120,7 +123,7 @@ jobs: py_version: type: string macos: - xcode: "13.4.1" + xcode: "15.3.0" steps: - macos: py_version: << parameters.py_version >> From 47fb80069966bc8d9c2a596c86eee52d4ab493c1 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Wed, 19 Jun 2024 13:44:30 +0800 Subject: [PATCH 07/14] . --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a574c10c9..557a2fd4ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,7 +48,11 @@ commands: command: | brew --version brew untap Homebrew/homebrew-cask-versions + # Adopt JDK issue workaround, may no longer needed in future MacOS base images + brew uninstall adoptopenjdk11 brew untap adoptopenjdk/openjdk + # brew install openjdk + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ brew update brew upgrade git brew install fish From cb2d9484cc19a96f6ca20faa2d422f71c9997ee0 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Wed, 19 Jun 2024 13:52:56 +0800 Subject: [PATCH 08/14] . --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 557a2fd4ab..e975ad54a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,11 +48,6 @@ commands: command: | brew --version brew untap Homebrew/homebrew-cask-versions - # Adopt JDK issue workaround, may no longer needed in future MacOS base images - brew uninstall adoptopenjdk11 - brew untap adoptopenjdk/openjdk - # brew install openjdk - # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ brew update brew upgrade git brew install fish From c720dd6771e9408f6d868bb6dda5bdec001d9489 Mon Sep 17 00:00:00 2001 From: Jasha Sommer-Simpson <8935917+Jasha10@users.noreply.github.com> Date: Wed, 19 Jun 2024 11:53:40 -0500 Subject: [PATCH 09/14] set -x to debug mac brew --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e975ad54a4..1c06012b76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,7 @@ commands: - run: name: Preparing environment - Brew command: | + set -x brew --version brew untap Homebrew/homebrew-cask-versions brew update From 24e7c1af62fa346dc0480a10ff58383d7b309758 Mon Sep 17 00:00:00 2001 From: Jasha Sommer-Simpson <8935917+Jasha10@users.noreply.github.com> Date: Thu, 20 Jun 2024 06:22:39 -0500 Subject: [PATCH 10/14] MacOS CI fix for Unexpected appcast call on adoptopenjdk11 Attempt solving MacOS CI Error: `Unexpected method 'appcast' called on Cask adoptopenjdk11.` Using tip from https://github.com/Homebrew/homebrew-cask/issues/173201#issuecomment-2101662209 --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c06012b76..3bdbce0c5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,7 @@ commands: set -x brew --version brew untap Homebrew/homebrew-cask-versions + rm -rf $(brew --caskroom)/adoptopenjdk11 # https://github.com/Homebrew/homebrew-cask/issues/173201#issuecomment-2101662209 brew update brew upgrade git brew install fish From 7e12f580385c3e735963eee1c996f6c6a1903a16 Mon Sep 17 00:00:00 2001 From: Jasha Sommer-Simpson <8935917+Jasha10@users.noreply.github.com> Date: Sat, 22 Jun 2024 16:35:20 -0500 Subject: [PATCH 11/14] config.yml: use macos M1 chipset instead of intel --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bdbce0c5e..47dedae997 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -125,6 +125,7 @@ jobs: type: string macos: xcode: "15.3.0" + resource_class: macos.m1.medium.gen1 steps: - macos: py_version: << parameters.py_version >> @@ -203,6 +204,7 @@ jobs: type: string macos: xcode: "13.4.1" + resource_class: macos.m1.medium.gen1 steps: - macos: py_version: << parameters.py_version >> From 4c2cfc41ae394272f204f1c1f84fa510b9422cfb Mon Sep 17 00:00:00 2001 From: Jasha Sommer-Simpson <8935917+Jasha10@users.noreply.github.com> Date: Sat, 22 Jun 2024 16:44:07 -0500 Subject: [PATCH 12/14] Update config.yml MacOSX miniconda installer --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47dedae997..118942af25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,8 +39,8 @@ commands: if [[ -f ~/miniconda3/LICENSE.txt ]] ; then echo "miniconda installed already." else - curl -o Miniconda3-py38_4.8.3-MacOSX-x86_64.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-MacOSX-x86_64.sh - bash ./Miniconda3-py38_4.8.3-MacOSX-x86_64.sh -b + curl -o Miniconda3-latest-MacOSX-arm64.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh + bash ./Miniconda3-latest-MacOSX-arm64.sh -b fi ~/miniconda3/bin/conda init bash - run: From be2261ae125c9644abedb0ccb0a5b57978238e2c Mon Sep 17 00:00:00 2001 From: Jasha Sommer-Simpson <8935917+Jasha10@users.noreply.github.com> Date: Sun, 23 Jun 2024 06:30:49 -0500 Subject: [PATCH 13/14] config.yml: debug MacOSX miniconda installation --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 118942af25..08c0c9f0ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,10 +36,12 @@ commands: - run: name: Preparing environment - Conda command: | + set -x if [[ -f ~/miniconda3/LICENSE.txt ]] ; then echo "miniconda installed already." else curl -o Miniconda3-latest-MacOSX-arm64.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh + uname -m bash ./Miniconda3-latest-MacOSX-arm64.sh -b fi ~/miniconda3/bin/conda init bash From f1863ec39a9e486e2e9da1fb22aac6c0f8ec54fa Mon Sep 17 00:00:00 2001 From: Jasha Sommer-Simpson <8935917+Jasha10@users.noreply.github.com> Date: Sun, 23 Jun 2024 06:46:13 -0500 Subject: [PATCH 14/14] circleci: cache based on OS/CPU architecture --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08c0c9f0ec..16668d3167 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ commands: steps: - checkout - restore_cache: - key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ .Branch }}-<< parameters.py_version >> + key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >> - run: name: Preparing environment - Conda command: | @@ -63,7 +63,7 @@ commands: conda create -n hydra python=<< parameters.py_version >> -yqc conda-forge conda run -n hydra pip install nox --progress-bar off - save_cache: - key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ .Branch }}-<< parameters.py_version >> + key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >> paths: - ~/miniconda3 - ~/Library/Caches/Homebrew @@ -101,7 +101,7 @@ commands: steps: - checkout - restore_cache: - key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ .Branch }}-<< parameters.py_version >> + key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >> - run: name: Preparing environment - system no_output_timeout: 20m @@ -116,7 +116,7 @@ commands: conda activate hydra pip install nox dataclasses --progress-bar off - save_cache: - key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ .Branch }}-<< parameters.py_version >> + key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >> paths: - C:\tools\miniconda3