From 5ad4eec260cc43ff944b2ce236040c5025d99a4a Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Wed, 31 Jan 2024 11:06:18 +0100 Subject: [PATCH 01/14] chore: wip --- py/h2o_wave/Makefile | 9 +++++++++ py/h2o_wave/conda/meta.yaml | 3 +++ py/h2o_wave/hatch_build.py | 12 ++++++++++-- py/h2o_wave/pyproject.toml | 12 +++++++++++- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index d0dbbec6f6..29ae424556 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -7,6 +7,15 @@ build: purge H2O_WAVE_PLATFORM=macosx_12_0_arm64 ../venv/bin/python3 -m build --wheel ../venv/bin/python3 -m build --wheel +build-conda: + mkdir -p conda/bin + cp -a ../../build/wave-1.0.0-darwin-arm64/. conda/bin + # rm -r conda/bin/examples + # rm -r conda/bin/demo + # rm -r conda/bin/test + VERSION=1.0.0 H2O_WAVE_PLATFORM=macosx_12_0_arm64 conda build --output-folder temp_build/ conda + rm -r conda/bin + purge: ## Purge previous build rm -f h2o_wave/metadata.py rm -rf build dist h2o_wave.egg-info \ No newline at end of file diff --git a/py/h2o_wave/conda/meta.yaml b/py/h2o_wave/conda/meta.yaml index d6ea34e3e6..5065c1461f 100644 --- a/py/h2o_wave/conda/meta.yaml +++ b/py/h2o_wave/conda/meta.yaml @@ -26,6 +26,9 @@ source: path: .. build: + script_env: + - VERSION + - H2O_WAVE_PLATFORM noarch: python script: python -m pip install . -vv entry_points: diff --git a/py/h2o_wave/hatch_build.py b/py/h2o_wave/hatch_build.py index b09ff8c631..7377e18273 100644 --- a/py/h2o_wave/hatch_build.py +++ b/py/h2o_wave/hatch_build.py @@ -17,12 +17,15 @@ def create_metadata_file(platform: str, arch: str): class CustomMetadataHook(MetadataHookInterface): def update(self, metadata: dict) -> None: + print('======> HATCH SETTING VERSION !!!! <======', os.environ.get('VERSION')) metadata['version'] = os.environ.get('VERSION', metadata['version']) class CustomBuildHook(BuildHookInterface): def initialize(self, _version, build_data): + print('======> HATCH !!!! <======') platform = os.environ.get('H2O_WAVE_PLATFORM') + print(f'Building for platform: {platform}') if not platform: # Create a default metadata file in case of noarch builds. create_metadata_file('linux', 'amd64') @@ -31,7 +34,7 @@ def initialize(self, _version, build_data): build_data['tag'] = f'py3-none-{platform}' build_data['pure_python'] = False - version = os.environ.get('VERSION') + version = os.environ.get('VERSION', '1.0.0') if not version: raise Exception('VERSION environment variable must be set.') @@ -42,12 +45,15 @@ def initialize(self, _version, build_data): elif platform == 'manylinux1_x86_64': operating_system = 'linux' - binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}') + print('==========>>> CWD:', os. getcwd()) + # binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}') + binaries_path = os.path.join('conda', 'bin') if not os.path.exists(binaries_path): raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.') self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test']) self.copy_files('project_templates', 'tmp', [], True) + shutil.rmtree(binaries_path, ignore_errors=True) # Only if conda!! create_metadata_file(operating_system, arch) @@ -63,3 +69,5 @@ def copy_files(self, src, dst, ignore, keep_dir=False) -> None: def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: shutil.rmtree('tmp', ignore_errors=True) + shutil.rmtree('info/recipe/bin', ignore_errors=True) # Conda only!! + diff --git a/py/h2o_wave/pyproject.toml b/py/h2o_wave/pyproject.toml index 90ae9ea79d..08b6b7c647 100644 --- a/py/h2o_wave/pyproject.toml +++ b/py/h2o_wave/pyproject.toml @@ -55,4 +55,14 @@ path = "hatch_build.py" [tool.hatch.metadata.hooks.custom] path = "hatch_build.py" [tool.hatch.build.targets.wheel.shared-data] -"tmp" = "." \ No newline at end of file +"tmp" = "./site-packages/h2o_wave-1.0.0.data" +# "tmp" = "." + +# [tool.hatch.build.hooks.custom] +# path = "hatch_build.py" +# [tool.hatch.build.shared-data] +# "tmp" = "." +# [tool.hatch.build.targets.sdist.hooks.custom] +# path = "hatch_build.py" +# [tool.hatch.build.targets.sdist.shared-data] +# "tmp" = "." \ No newline at end of file From 64d77549c4b3b45358d8ab48d904cdd8d6fd17fc Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Thu, 15 Feb 2024 17:49:46 +0100 Subject: [PATCH 02/14] feat: use bash script to move binary to the python exec path after install --- py/h2o_wave/Makefile | 16 ++++++++-------- py/h2o_wave/conda/post-link.bat | 0 py/h2o_wave/conda/post-link.sh | 17 +++++++++++++++++ py/h2o_wave/conda/pre-unlink.bat | 0 py/h2o_wave/conda/pre-unlink.sh | 10 ++++++++++ py/h2o_wave/hatch_build.py | 5 +++-- py/h2o_wave/pyproject.toml | 2 +- 7 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 py/h2o_wave/conda/post-link.bat create mode 100644 py/h2o_wave/conda/post-link.sh create mode 100644 py/h2o_wave/conda/pre-unlink.bat create mode 100644 py/h2o_wave/conda/pre-unlink.sh diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index 29ae424556..599ce14ddc 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -7,14 +7,14 @@ build: purge H2O_WAVE_PLATFORM=macosx_12_0_arm64 ../venv/bin/python3 -m build --wheel ../venv/bin/python3 -m build --wheel -build-conda: - mkdir -p conda/bin - cp -a ../../build/wave-1.0.0-darwin-arm64/. conda/bin - # rm -r conda/bin/examples - # rm -r conda/bin/demo - # rm -r conda/bin/test - VERSION=1.0.0 H2O_WAVE_PLATFORM=macosx_12_0_arm64 conda build --output-folder temp_build/ conda - rm -r conda/bin +build-conda: purge + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-darwin-arm64/. conda/temp + # rm -r conda/temp/examples + # rm -r conda/temp/demo + # rm -r conda/temp/test + VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_12_0_arm64 conda build --output-folder temp_build/ conda + rm -r conda/temp purge: ## Purge previous build rm -f h2o_wave/metadata.py diff --git a/py/h2o_wave/conda/post-link.bat b/py/h2o_wave/conda/post-link.bat new file mode 100644 index 0000000000..e69de29bb2 diff --git a/py/h2o_wave/conda/post-link.sh b/py/h2o_wave/conda/post-link.sh new file mode 100644 index 0000000000..96d64d295c --- /dev/null +++ b/py/h2o_wave/conda/post-link.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Moves waved with its dependencies to the python environment + +PACKAGE_NAME=${PKG_NAME/-/_} + +# Loop through all files/folders starting with "python" in the lib/$PREFIX directory +for item in $PREFIX/lib/python*; do + # Check if it's a directory and "/site-packages/$PACKAGE_NAME-$PKG_VERSION.data" exists inside + if [[ -d "$item" && -d "$item/site-packages/$PACKAGE_NAME-$PKG_VERSION.data" ]]; then + # Move the folder content to the python environment + mv -v $item/site-packages/$PACKAGE_NAME-$PKG_VERSION.data/* $PREFIX/ + # Remove data from old location. + rm -rf $item/site-packages/$PACKAGE_NAME-$PKG_VERSION.data + # Break the loop + break + fi +done \ No newline at end of file diff --git a/py/h2o_wave/conda/pre-unlink.bat b/py/h2o_wave/conda/pre-unlink.bat new file mode 100644 index 0000000000..e69de29bb2 diff --git a/py/h2o_wave/conda/pre-unlink.sh b/py/h2o_wave/conda/pre-unlink.sh new file mode 100644 index 0000000000..38277cb979 --- /dev/null +++ b/py/h2o_wave/conda/pre-unlink.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Removes folders "project_templates", "www" and file "waved" from the python environment on uninstall + +PACKAGE_NAME=${PKG_NAME/-/_} + +rm -rf $PREFIX/project_templates +rm -rf $PREFIX/www +rm -f $PREFIX/waved + + diff --git a/py/h2o_wave/hatch_build.py b/py/h2o_wave/hatch_build.py index 7377e18273..c7356b3d90 100644 --- a/py/h2o_wave/hatch_build.py +++ b/py/h2o_wave/hatch_build.py @@ -1,5 +1,6 @@ import os import shutil +import sys from typing import Any, Dict from hatchling.builders.hooks.plugin.interface import BuildHookInterface @@ -47,7 +48,7 @@ def initialize(self, _version, build_data): print('==========>>> CWD:', os. getcwd()) # binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}') - binaries_path = os.path.join('conda', 'bin') + binaries_path = os.path.join('conda', 'temp') if not os.path.exists(binaries_path): raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.') @@ -69,5 +70,5 @@ def copy_files(self, src, dst, ignore, keep_dir=False) -> None: def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: shutil.rmtree('tmp', ignore_errors=True) - shutil.rmtree('info/recipe/bin', ignore_errors=True) # Conda only!! + shutil.rmtree('info/recipe/temp', ignore_errors=True) # Conda only!! diff --git a/py/h2o_wave/pyproject.toml b/py/h2o_wave/pyproject.toml index 08b6b7c647..87633a5056 100644 --- a/py/h2o_wave/pyproject.toml +++ b/py/h2o_wave/pyproject.toml @@ -55,7 +55,7 @@ path = "hatch_build.py" [tool.hatch.metadata.hooks.custom] path = "hatch_build.py" [tool.hatch.build.targets.wheel.shared-data] -"tmp" = "./site-packages/h2o_wave-1.0.0.data" +"tmp" = "./site-packages/h2o_wave-1.0.3.data" # "tmp" = "." # [tool.hatch.build.hooks.custom] From 3bd4f6a6c24b3e359af266e2f44fc49afb88b1b3 Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Fri, 16 Feb 2024 14:53:13 +0100 Subject: [PATCH 03/14] chore: build for every platform individually --- py/.gitignore | 1 + py/h2o_wave/Makefile | 35 +++++++++++++++++++++++++++++++---- py/h2o_wave/hatch_build.py | 5 +---- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/py/.gitignore b/py/.gitignore index ec60ae5112..f33b0fe04e 100644 --- a/py/.gitignore +++ b/py/.gitignore @@ -9,6 +9,7 @@ __pycache__/ # Distribution / packaging .Python build/ +build-conda/ develop-eggs/ dist/ downloads/ diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index 599ce14ddc..3ce4a2cc84 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -8,13 +8,40 @@ build: purge ../venv/bin/python3 -m build --wheel build-conda: purge + rm -rf temp_build ../build-conda/ +# Mac arm64 mkdir -p conda/temp +# rm -r conda/temp/examples conda/temp/demo conda/temp/test cp -a ../../build/wave-1.0.3-darwin-arm64/. conda/temp - # rm -r conda/temp/examples - # rm -r conda/temp/demo - # rm -r conda/temp/test VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_12_0_arm64 conda build --output-folder temp_build/ conda - rm -r conda/temp + mkdir -p ../build-conda/osx-arm64 + mv temp_build/noarch/*.tar.bz2 ../build-conda/osx-arm64 + rm -rf temp_build conda/temp + +# Mac x86_64 + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-darwin-amd64/. conda/temp + VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_10_9_x86_64 conda build --output-folder temp_build/ conda + mkdir -p ../build-conda/osx-64 + mv temp_build/noarch/*.tar.bz2 ../build-conda/osx-64 + rm -rf temp_build conda/temp + +# Linux 64 (the same as 32) + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-linux-amd64/. conda/temp + VERSION=1.0.3 H2O_WAVE_PLATFORM=manylinux1_x86_64 conda build --output-folder temp_build/ conda + mkdir -p ../build-conda/linux-64 + mv temp_build/noarch/*.tar.bz2 ../build-conda/linux-64 + rm -rf temp_build conda/temp + +# Windows 64 (TODO 32) + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-windows-amd64/. conda/temp + VERSION=1.0.3 H2O_WAVE_PLATFORM=win_amd64 conda build --output-folder temp_build/ conda + mkdir -p ../build-conda/win-64 + mv temp_build/noarch/*.tar.bz2 ../build-conda/win-64 + rm -rf temp_build conda/temp + purge: ## Purge previous build rm -f h2o_wave/metadata.py diff --git a/py/h2o_wave/hatch_build.py b/py/h2o_wave/hatch_build.py index c7356b3d90..81f9d6fd0c 100644 --- a/py/h2o_wave/hatch_build.py +++ b/py/h2o_wave/hatch_build.py @@ -18,13 +18,11 @@ def create_metadata_file(platform: str, arch: str): class CustomMetadataHook(MetadataHookInterface): def update(self, metadata: dict) -> None: - print('======> HATCH SETTING VERSION !!!! <======', os.environ.get('VERSION')) metadata['version'] = os.environ.get('VERSION', metadata['version']) class CustomBuildHook(BuildHookInterface): def initialize(self, _version, build_data): - print('======> HATCH !!!! <======') platform = os.environ.get('H2O_WAVE_PLATFORM') print(f'Building for platform: {platform}') if not platform: @@ -35,7 +33,7 @@ def initialize(self, _version, build_data): build_data['tag'] = f'py3-none-{platform}' build_data['pure_python'] = False - version = os.environ.get('VERSION', '1.0.0') + version = os.environ.get('VERSION') if not version: raise Exception('VERSION environment variable must be set.') @@ -46,7 +44,6 @@ def initialize(self, _version, build_data): elif platform == 'manylinux1_x86_64': operating_system = 'linux' - print('==========>>> CWD:', os. getcwd()) # binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}') binaries_path = os.path.join('conda', 'temp') if not os.path.exists(binaries_path): From 5ac9916fff8e121fa6c43a4f7a1ba09260bfec01 Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Tue, 20 Feb 2024 13:16:08 +0100 Subject: [PATCH 04/14] chore: get rid of link scripts --- py/h2o_wave/Makefile | 51 ++++++++++++++++---------------- py/h2o_wave/conda/meta.yaml | 6 ++-- py/h2o_wave/conda/post-link.bat | 0 py/h2o_wave/conda/post-link.sh | 17 ----------- py/h2o_wave/conda/pre-unlink.bat | 0 py/h2o_wave/conda/pre-unlink.sh | 10 ------- py/h2o_wave/hatch_build.py | 20 +++++++------ py/h2o_wave/pyproject.toml | 12 +------- 8 files changed, 39 insertions(+), 77 deletions(-) delete mode 100644 py/h2o_wave/conda/post-link.bat delete mode 100644 py/h2o_wave/conda/post-link.sh delete mode 100644 py/h2o_wave/conda/pre-unlink.bat delete mode 100644 py/h2o_wave/conda/pre-unlink.sh diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index 3ce4a2cc84..8024717ab7 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -8,39 +8,38 @@ build: purge ../venv/bin/python3 -m build --wheel build-conda: purge - rm -rf temp_build ../build-conda/ + rm -rf temp_build ../sdist/ # Mac arm64 mkdir -p conda/temp -# rm -r conda/temp/examples conda/temp/demo conda/temp/test cp -a ../../build/wave-1.0.3-darwin-arm64/. conda/temp - VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_12_0_arm64 conda build --output-folder temp_build/ conda - mkdir -p ../build-conda/osx-arm64 - mv temp_build/noarch/*.tar.bz2 ../build-conda/osx-arm64 + VERSION=1.0.3 conda build --output-folder temp_build/ conda + mkdir -p ../sdist/osx-arm64 + mv temp_build/noarch/*.tar.bz2 ../sdist/osx-arm64 rm -rf temp_build conda/temp -# Mac x86_64 - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-darwin-amd64/. conda/temp - VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_10_9_x86_64 conda build --output-folder temp_build/ conda - mkdir -p ../build-conda/osx-64 - mv temp_build/noarch/*.tar.bz2 ../build-conda/osx-64 - rm -rf temp_build conda/temp +# # Mac x86_64 +# mkdir -p conda/temp +# cp -a ../../build/wave-1.0.3-darwin-amd64/. conda/temp +# VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_10_9_x86_64 conda build --output-folder temp_build/ conda +# mkdir -p ../sdist/osx-64 +# mv temp_build/noarch/*.tar.bz2 ../sdist/osx-64 +# rm -rf temp_build conda/temp -# Linux 64 (the same as 32) - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-linux-amd64/. conda/temp - VERSION=1.0.3 H2O_WAVE_PLATFORM=manylinux1_x86_64 conda build --output-folder temp_build/ conda - mkdir -p ../build-conda/linux-64 - mv temp_build/noarch/*.tar.bz2 ../build-conda/linux-64 - rm -rf temp_build conda/temp +# # Linux 64 (the same as 32) +# mkdir -p conda/temp +# cp -a ../../build/wave-1.0.3-linux-amd64/. conda/temp +# VERSION=1.0.3 H2O_WAVE_PLATFORM=manylinux1_x86_64 conda build --output-folder temp_build/ conda +# mkdir -p ../sdist/linux-64 +# mv temp_build/noarch/*.tar.bz2 ../sdist/linux-64 +# rm -rf temp_build conda/temp -# Windows 64 (TODO 32) - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-windows-amd64/. conda/temp - VERSION=1.0.3 H2O_WAVE_PLATFORM=win_amd64 conda build --output-folder temp_build/ conda - mkdir -p ../build-conda/win-64 - mv temp_build/noarch/*.tar.bz2 ../build-conda/win-64 - rm -rf temp_build conda/temp +# # Windows 64 (TODO 32) +# mkdir -p conda/temp +# cp -a ../../build/wave-1.0.3-windows-amd64/. conda/temp +# VERSION=1.0.3 H2O_WAVE_PLATFORM=win_amd64 conda build --output-folder temp_build/ conda +# mkdir -p ../sdist/win-64 +# mv temp_build/noarch/*.tar.bz2 ../sdist/win-64 +# rm -rf temp_build conda/temp purge: ## Purge previous build diff --git a/py/h2o_wave/conda/meta.yaml b/py/h2o_wave/conda/meta.yaml index 5065c1461f..c152109ded 100644 --- a/py/h2o_wave/conda/meta.yaml +++ b/py/h2o_wave/conda/meta.yaml @@ -26,12 +26,10 @@ source: path: .. build: - script_env: - - VERSION - - H2O_WAVE_PLATFORM noarch: python script: python -m pip install . -vv entry_points: {% for module, entrypoints in project['scripts'].items() %} - {{ module }} = {{ entrypoints }} - {% endfor %} \ No newline at end of file + {% endfor %} + include_recipe: False \ No newline at end of file diff --git a/py/h2o_wave/conda/post-link.bat b/py/h2o_wave/conda/post-link.bat deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/py/h2o_wave/conda/post-link.sh b/py/h2o_wave/conda/post-link.sh deleted file mode 100644 index 96d64d295c..0000000000 --- a/py/h2o_wave/conda/post-link.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Moves waved with its dependencies to the python environment - -PACKAGE_NAME=${PKG_NAME/-/_} - -# Loop through all files/folders starting with "python" in the lib/$PREFIX directory -for item in $PREFIX/lib/python*; do - # Check if it's a directory and "/site-packages/$PACKAGE_NAME-$PKG_VERSION.data" exists inside - if [[ -d "$item" && -d "$item/site-packages/$PACKAGE_NAME-$PKG_VERSION.data" ]]; then - # Move the folder content to the python environment - mv -v $item/site-packages/$PACKAGE_NAME-$PKG_VERSION.data/* $PREFIX/ - # Remove data from old location. - rm -rf $item/site-packages/$PACKAGE_NAME-$PKG_VERSION.data - # Break the loop - break - fi -done \ No newline at end of file diff --git a/py/h2o_wave/conda/pre-unlink.bat b/py/h2o_wave/conda/pre-unlink.bat deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/py/h2o_wave/conda/pre-unlink.sh b/py/h2o_wave/conda/pre-unlink.sh deleted file mode 100644 index 38277cb979..0000000000 --- a/py/h2o_wave/conda/pre-unlink.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Removes folders "project_templates", "www" and file "waved" from the python environment on uninstall - -PACKAGE_NAME=${PKG_NAME/-/_} - -rm -rf $PREFIX/project_templates -rm -rf $PREFIX/www -rm -f $PREFIX/waved - - diff --git a/py/h2o_wave/hatch_build.py b/py/h2o_wave/hatch_build.py index 81f9d6fd0c..e79573251c 100644 --- a/py/h2o_wave/hatch_build.py +++ b/py/h2o_wave/hatch_build.py @@ -24,10 +24,12 @@ def update(self, metadata: dict) -> None: class CustomBuildHook(BuildHookInterface): def initialize(self, _version, build_data): platform = os.environ.get('H2O_WAVE_PLATFORM') - print(f'Building for platform: {platform}') if not platform: # Create a default metadata file in case of noarch builds. create_metadata_file('linux', 'amd64') + # If conda noarch build, copy binaries into package. + if os.environ.get('CONDA_BUILD'): + self.copy_binaries_into_package(os.path.join('conda', 'temp')) return build_data['tag'] = f'py3-none-{platform}' @@ -44,14 +46,7 @@ def initialize(self, _version, build_data): elif platform == 'manylinux1_x86_64': operating_system = 'linux' - # binaries_path = os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}') - binaries_path = os.path.join('conda', 'temp') - if not os.path.exists(binaries_path): - raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.') - - self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test']) - self.copy_files('project_templates', 'tmp', [], True) - shutil.rmtree(binaries_path, ignore_errors=True) # Only if conda!! + self.copy_binaries_into_package(os.path.join('..', '..', 'build', f'wave-{version}-{operating_system}-{arch}')) create_metadata_file(operating_system, arch) @@ -65,6 +60,13 @@ def copy_files(self, src, dst, ignore, keep_dir=False) -> None: elif os.path.isdir(src_file) and file_name not in ignore: self.copy_files(src_file, dst_file, ignore) + def copy_binaries_into_package(self, binaries_path): + if not os.path.exists(binaries_path): + raise Exception(f'{binaries_path} does not exist. Run make release first to generate server binaries.') + + self.copy_files(binaries_path, 'tmp', ['demo', 'examples', 'test']) + self.copy_files('project_templates', 'tmp', [], True) + def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: shutil.rmtree('tmp', ignore_errors=True) shutil.rmtree('info/recipe/temp', ignore_errors=True) # Conda only!! diff --git a/py/h2o_wave/pyproject.toml b/py/h2o_wave/pyproject.toml index 87633a5056..90ae9ea79d 100644 --- a/py/h2o_wave/pyproject.toml +++ b/py/h2o_wave/pyproject.toml @@ -55,14 +55,4 @@ path = "hatch_build.py" [tool.hatch.metadata.hooks.custom] path = "hatch_build.py" [tool.hatch.build.targets.wheel.shared-data] -"tmp" = "./site-packages/h2o_wave-1.0.3.data" -# "tmp" = "." - -# [tool.hatch.build.hooks.custom] -# path = "hatch_build.py" -# [tool.hatch.build.shared-data] -# "tmp" = "." -# [tool.hatch.build.targets.sdist.hooks.custom] -# path = "hatch_build.py" -# [tool.hatch.build.targets.sdist.shared-data] -# "tmp" = "." \ No newline at end of file +"tmp" = "." \ No newline at end of file From 29c8c50245f9d0264354659444017838d08a1254 Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Tue, 20 Feb 2024 13:19:50 +0100 Subject: [PATCH 05/14] chore: remove leftovers --- py/h2o_wave/Makefile | 42 +++++++++++++++++++------------------- py/h2o_wave/hatch_build.py | 2 -- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index 8024717ab7..39f8d7890b 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -17,29 +17,29 @@ build-conda: purge mv temp_build/noarch/*.tar.bz2 ../sdist/osx-arm64 rm -rf temp_build conda/temp -# # Mac x86_64 -# mkdir -p conda/temp -# cp -a ../../build/wave-1.0.3-darwin-amd64/. conda/temp -# VERSION=1.0.3 H2O_WAVE_PLATFORM=macosx_10_9_x86_64 conda build --output-folder temp_build/ conda -# mkdir -p ../sdist/osx-64 -# mv temp_build/noarch/*.tar.bz2 ../sdist/osx-64 -# rm -rf temp_build conda/temp +# Mac x86_64 + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-darwin-amd64/. conda/temp + VERSION=1.0.3 conda build --output-folder temp_build/ conda + mkdir -p ../sdist/osx-64 + mv temp_build/noarch/*.tar.bz2 ../sdist/osx-64 + rm -rf temp_build conda/temp -# # Linux 64 (the same as 32) -# mkdir -p conda/temp -# cp -a ../../build/wave-1.0.3-linux-amd64/. conda/temp -# VERSION=1.0.3 H2O_WAVE_PLATFORM=manylinux1_x86_64 conda build --output-folder temp_build/ conda -# mkdir -p ../sdist/linux-64 -# mv temp_build/noarch/*.tar.bz2 ../sdist/linux-64 -# rm -rf temp_build conda/temp +# Linux 64 (the same as 32) + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-linux-amd64/. conda/temp + VERSION=1.0.3 conda build --output-folder temp_build/ conda + mkdir -p ../sdist/linux-64 + mv temp_build/noarch/*.tar.bz2 ../sdist/linux-64 + rm -rf temp_build conda/temp -# # Windows 64 (TODO 32) -# mkdir -p conda/temp -# cp -a ../../build/wave-1.0.3-windows-amd64/. conda/temp -# VERSION=1.0.3 H2O_WAVE_PLATFORM=win_amd64 conda build --output-folder temp_build/ conda -# mkdir -p ../sdist/win-64 -# mv temp_build/noarch/*.tar.bz2 ../sdist/win-64 -# rm -rf temp_build conda/temp +# Windows 64 (TODO 32) + mkdir -p conda/temp + cp -a ../../build/wave-1.0.3-windows-amd64/. conda/temp + VERSION=1.0.3 conda build --output-folder temp_build/ conda + mkdir -p ../sdist/win-64 + mv temp_build/noarch/*.tar.bz2 ../sdist/win-64 + rm -rf temp_build conda/temp purge: ## Purge previous build diff --git a/py/h2o_wave/hatch_build.py b/py/h2o_wave/hatch_build.py index e79573251c..9252275ebe 100644 --- a/py/h2o_wave/hatch_build.py +++ b/py/h2o_wave/hatch_build.py @@ -1,6 +1,5 @@ import os import shutil -import sys from typing import Any, Dict from hatchling.builders.hooks.plugin.interface import BuildHookInterface @@ -69,5 +68,4 @@ def copy_binaries_into_package(self, binaries_path): def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: shutil.rmtree('tmp', ignore_errors=True) - shutil.rmtree('info/recipe/temp', ignore_errors=True) # Conda only!! From 104a0741bd4c8e07b447b1a07898e27dced79fde Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Tue, 20 Feb 2024 17:53:47 +0100 Subject: [PATCH 06/14] chore: refactor makefile --- py/.gitignore | 1 - py/h2o_wave/Makefile | 53 ++++++++++++-------------------------- py/h2o_wave/hatch_build.py | 7 ++--- 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/py/.gitignore b/py/.gitignore index f33b0fe04e..ec60ae5112 100644 --- a/py/.gitignore +++ b/py/.gitignore @@ -9,7 +9,6 @@ __pycache__/ # Distribution / packaging .Python build/ -build-conda/ develop-eggs/ dist/ downloads/ diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index 39f8d7890b..b675619d85 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -1,3 +1,13 @@ +build_conda_package = \ + if [ "$(1)" != "noarch" ]; then \ + mkdir -p conda/temp && \ + cp -a ../../build/wave-$(VERSION)-$(1)/. conda/temp; \ + fi && \ + conda build --output-folder temp_build/ conda && \ + mkdir -p sdist/$(2) && \ + mv temp_build/noarch/*.tar.bz2 sdist/$(2) && \ + rm -rf temp_build conda/temp + .PHONY: build build: purge H2O_WAVE_PLATFORM=win_amd64 ../venv/bin/python3 -m build --wheel @@ -6,42 +16,11 @@ build: purge H2O_WAVE_PLATFORM=macosx_11_0_arm64 ../venv/bin/python3 -m build --wheel H2O_WAVE_PLATFORM=macosx_12_0_arm64 ../venv/bin/python3 -m build --wheel ../venv/bin/python3 -m build --wheel - -build-conda: purge - rm -rf temp_build ../sdist/ -# Mac arm64 - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-darwin-arm64/. conda/temp - VERSION=1.0.3 conda build --output-folder temp_build/ conda - mkdir -p ../sdist/osx-arm64 - mv temp_build/noarch/*.tar.bz2 ../sdist/osx-arm64 - rm -rf temp_build conda/temp - -# Mac x86_64 - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-darwin-amd64/. conda/temp - VERSION=1.0.3 conda build --output-folder temp_build/ conda - mkdir -p ../sdist/osx-64 - mv temp_build/noarch/*.tar.bz2 ../sdist/osx-64 - rm -rf temp_build conda/temp - -# Linux 64 (the same as 32) - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-linux-amd64/. conda/temp - VERSION=1.0.3 conda build --output-folder temp_build/ conda - mkdir -p ../sdist/linux-64 - mv temp_build/noarch/*.tar.bz2 ../sdist/linux-64 - rm -rf temp_build conda/temp - -# Windows 64 (TODO 32) - mkdir -p conda/temp - cp -a ../../build/wave-1.0.3-windows-amd64/. conda/temp - VERSION=1.0.3 conda build --output-folder temp_build/ conda - mkdir -p ../sdist/win-64 - mv temp_build/noarch/*.tar.bz2 ../sdist/win-64 - rm -rf temp_build conda/temp - + $(call build_conda_package,darwin-arm64,osx-arm64) + $(call build_conda_package,darwin-amd64,osx-64) + $(call build_conda_package,linux-amd64,linux-64) + $(call build_conda_package,windows-amd64,win-64) + $(call build_conda_package,noarch,noarch) purge: ## Purge previous build - rm -f h2o_wave/metadata.py - rm -rf build dist h2o_wave.egg-info \ No newline at end of file + rm -rf build dist sdist h2o_wave.egg-info temp_build conda/temp h2o_wave/metadata.py \ No newline at end of file diff --git a/py/h2o_wave/hatch_build.py b/py/h2o_wave/hatch_build.py index 9252275ebe..2a81e50f3c 100644 --- a/py/h2o_wave/hatch_build.py +++ b/py/h2o_wave/hatch_build.py @@ -26,9 +26,10 @@ def initialize(self, _version, build_data): if not platform: # Create a default metadata file in case of noarch builds. create_metadata_file('linux', 'amd64') - # If conda noarch build, copy binaries into package. - if os.environ.get('CONDA_BUILD'): - self.copy_binaries_into_package(os.path.join('conda', 'temp')) + # If conda build, copy binaries into package. + binaries_path = os.path.join('conda', 'temp') + if os.environ.get('CONDA_BUILD') and os.path.exists(binaries_path): + self.copy_binaries_into_package(binaries_path) return build_data['tag'] = f'py3-none-{platform}' From ec440f40814d47357f4a7a23cce92d9daa86bda9 Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Thu, 7 Mar 2024 11:22:24 +0100 Subject: [PATCH 07/14] chore: add todo with current status --- py/h2o_wave/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index b675619d85..34c8ded2fc 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -23,4 +23,6 @@ build: purge $(call build_conda_package,noarch,noarch) purge: ## Purge previous build - rm -rf build dist sdist h2o_wave.egg-info temp_build conda/temp h2o_wave/metadata.py \ No newline at end of file + rm -rf build dist sdist h2o_wave.egg-info temp_build conda/temp h2o_wave/metadata.py + +# TODO: Finish once this is resolved: https://github.com/conda/conda-build/issues/2611#issuecomment-1239338538 \ No newline at end of file From e05d4115ca5f5bf2d3ebceb1b2de8ae3dd9833aa Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Tue, 26 Mar 2024 16:42:46 +0100 Subject: [PATCH 08/14] feat: build binary included conda package for each platform and python version separately --- py/h2o_wave/Makefile | 14 +++++++++----- py/h2o_wave/conda/conda_build_config.yaml | 7 +++++++ py/h2o_wave/conda/meta.yaml | 16 ++++++++++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 py/h2o_wave/conda/conda_build_config.yaml diff --git a/py/h2o_wave/Makefile b/py/h2o_wave/Makefile index 34c8ded2fc..fa6d9316f2 100644 --- a/py/h2o_wave/Makefile +++ b/py/h2o_wave/Makefile @@ -1,11 +1,17 @@ build_conda_package = \ if [ "$(1)" != "noarch" ]; then \ - mkdir -p conda/temp && \ + mkdir -p conda/temp; \ cp -a ../../build/wave-$(VERSION)-$(1)/. conda/temp; \ + NO_ARCH=0 conda build --output-folder temp_build/ conda; \ + find temp_build/ -name *.tar.bz2 | while read -r file; do \ + conda convert --force --platform $(2) "$$file" -o temp_build/; \ + done \ + fi && \ + if [ "$(1)" == "noarch" ]; then \ + NO_ARCH=1 conda build --output-folder temp_build/ conda; \ fi && \ - conda build --output-folder temp_build/ conda && \ mkdir -p sdist/$(2) && \ - mv temp_build/noarch/*.tar.bz2 sdist/$(2) && \ + mv temp_build/$(2)/*.tar.bz2 sdist/$(2) && \ rm -rf temp_build conda/temp .PHONY: build @@ -24,5 +30,3 @@ build: purge purge: ## Purge previous build rm -rf build dist sdist h2o_wave.egg-info temp_build conda/temp h2o_wave/metadata.py - -# TODO: Finish once this is resolved: https://github.com/conda/conda-build/issues/2611#issuecomment-1239338538 \ No newline at end of file diff --git a/py/h2o_wave/conda/conda_build_config.yaml b/py/h2o_wave/conda/conda_build_config.yaml new file mode 100644 index 0000000000..5c1b78c223 --- /dev/null +++ b/py/h2o_wave/conda/conda_build_config.yaml @@ -0,0 +1,7 @@ +# TODO: Build a single package for all python versions once this is resolved: https://github.com/conda/conda-build/issues/2611#issuecomment-1239338538 +python_version: + - 3.9 + - 3.10 + - 3.11 + - 3.12 + \ No newline at end of file diff --git a/py/h2o_wave/conda/meta.yaml b/py/h2o_wave/conda/meta.yaml index c152109ded..7a3f640dc6 100644 --- a/py/h2o_wave/conda/meta.yaml +++ b/py/h2o_wave/conda/meta.yaml @@ -15,10 +15,16 @@ about: requirements: build: + {% if NO_ARCH == "0" %} + - python={{ python_version }} + {% endif %} {% for dep in pyproject['build-system']['requires'] %} - {{ dep.lower() }} {% endfor %} run: + {% if NO_ARCH == "0" %} + - python={{ python_version }} + {% endif %} {% for dep in project['dependencies'] %} - {{ dep.lower() }} {% endfor %} @@ -26,10 +32,16 @@ source: path: .. build: - noarch: python script: python -m pip install . -vv + string: py{{ python_version | replace(".", "") }} entry_points: {% for module, entrypoints in project['scripts'].items() %} - {{ module }} = {{ entrypoints }} {% endfor %} - include_recipe: False \ No newline at end of file + include_recipe: False + script_env: + - NO_ARCH + {% if NO_ARCH == "1" %} + noarch: python + string: "0" + {% endif %} From f4ec2f932cbbf8da0b9f84574f1da6f2e8b04e0f Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Tue, 26 Mar 2024 18:18:42 +0100 Subject: [PATCH 09/14] feat: update workflow and gh action --- .github/workflows/publish-conda.yml | 9 ++------- .github/workflows/release-wave.yml | 8 ++------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-conda.yml b/.github/workflows/publish-conda.yml index 28ad0d968b..f78e868c5b 100644 --- a/.github/workflows/publish-conda.yml +++ b/.github/workflows/publish-conda.yml @@ -9,7 +9,6 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "${{ github.event.inputs.version }}" jobs: release: @@ -21,12 +20,8 @@ jobs: token: ${{ secrets.GIT_TOKEN }} - name: Publish to Conda - uses: marek-mihok/publish_conda_package_action@v1.0.1 + uses: marek-mihok/upload_to_anaconda_cloud_action@v1.0.0 with: - CondaDir: 'py/h2o_wave/conda' - Platforms: 'noarch' + PackagesDir: 'py/h2o_wave/sdist' CondaUsername: ${{ secrets.CONDA_USERNAME }} CondaPassword: ${{ secrets.CONDA_PASSWORD }} - env: - # VERSION clashes with conda build. Use PKG_VERSION instead. - PKG_VERSION: ${{ env.VERSION }} diff --git a/.github/workflows/release-wave.yml b/.github/workflows/release-wave.yml index a3b37f3c44..ef100315bc 100644 --- a/.github/workflows/release-wave.yml +++ b/.github/workflows/release-wave.yml @@ -79,15 +79,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish to Conda - uses: marek-mihok/publish_conda_package_action@v1.0.1 + uses: marek-mihok/upload_to_anaconda_cloud_action@v1.0.0 with: - CondaDir: 'py/h2o_wave/conda' - Platforms: 'noarch' + PackagesDir: 'py/h2o_wave/sdist' CondaUsername: ${{ secrets.CONDA_USERNAME }} CondaPassword: ${{ secrets.CONDA_PASSWORD }} - env: - # VERSION clashes with conda build. Use PKG_VERSION instead. - PKG_VERSION: ${{ env.VERSION }} - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@master From 03c8e4807d59323c8cf49d10acc3a3f639ef9dea Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Tue, 26 Mar 2024 19:17:15 +0100 Subject: [PATCH 10/14] feat: add conda into gh action --- .github/workflows/publish-conda.yml | 15 +++++++++++++++ .github/workflows/release-wave.yml | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/publish-conda.yml b/.github/workflows/publish-conda.yml index f78e868c5b..478b36de70 100644 --- a/.github/workflows/publish-conda.yml +++ b/.github/workflows/publish-conda.yml @@ -19,6 +19,21 @@ jobs: with: token: ${{ secrets.GIT_TOKEN }} + - name: Setup + run: make setup + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: false + activate-environment: "" + conda-build-version: '24.3.0' + + - name: Build Release + run: make release + env: + NODE_OPTIONS: '--max-old-space-size=8192' + - name: Publish to Conda uses: marek-mihok/upload_to_anaconda_cloud_action@v1.0.0 with: diff --git a/.github/workflows/release-wave.yml b/.github/workflows/release-wave.yml index ef100315bc..f02fe303c9 100644 --- a/.github/workflows/release-wave.yml +++ b/.github/workflows/release-wave.yml @@ -41,6 +41,13 @@ jobs: with: working-directory: ./r + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: false + activate-environment: "" + conda-build-version: '24.3.0' + - name: Build Release run: make release env: From b417bacf186491fb296fc1764f016cb16606d46b Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Wed, 27 Mar 2024 12:09:40 +0100 Subject: [PATCH 11/14] chore: update docs --- website/docs/installation.mdx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/website/docs/installation.mdx b/website/docs/installation.mdx index 207c3c45ad..d1d350b189 100644 --- a/website/docs/installation.mdx +++ b/website/docs/installation.mdx @@ -22,9 +22,6 @@ pip install h2o-wave conda config --append channels conda-forge conda install -c h2oai h2o-wave ``` -:::info -Conda packaging does not contain Wave server which means, you will still need to run the wave server (waved) [separately](/docs/tutorial-hello#step-1-start-the-wave-server). -::: @@ -64,9 +61,6 @@ conda config --append channels conda-forge conda install -c h2oai h2o-wave ``` -:::info -Conda packaging does not contain Wave server which means, you will still need to run the wave server (waved) [separately](/docs/tutorial-hello#step-1-start-the-wave-server). -::: From 5e2fed3393bfed8b0ef8603b838272c62e21e3c2 Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Wed, 27 Mar 2024 12:15:02 +0100 Subject: [PATCH 12/14] chore: update publish conda only workflow --- .github/workflows/publish-conda.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/publish-conda.yml b/.github/workflows/publish-conda.yml index 478b36de70..12edce6ba4 100644 --- a/.github/workflows/publish-conda.yml +++ b/.github/workflows/publish-conda.yml @@ -19,9 +19,22 @@ jobs: with: token: ${{ secrets.GIT_TOKEN }} + - uses: actions/setup-go@v1 + with: + go-version: '1.19.4' + + - uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + - name: Setup run: make setup + - uses: r-lib/actions/setup-r@v2 + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: ./r + - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v3 with: From de74e9155fe146cedfd86fb21ac817a5de01065e Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Wed, 27 Mar 2024 12:21:09 +0100 Subject: [PATCH 13/14] chore: use PKG_VERSION instead of VERSION for conda build --- .github/workflows/publish-conda.yml | 4 +++- .github/workflows/release-wave.yml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-conda.yml b/.github/workflows/publish-conda.yml index 12edce6ba4..701fd1a11c 100644 --- a/.github/workflows/publish-conda.yml +++ b/.github/workflows/publish-conda.yml @@ -8,7 +8,7 @@ on: required: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: "${{ github.event.inputs.version }}" jobs: release: @@ -45,6 +45,8 @@ jobs: - name: Build Release run: make release env: + # VERSION clashes with conda build. Use PKG_VERSION instead. + PKG_VERSION: ${{ env.VERSION }} NODE_OPTIONS: '--max-old-space-size=8192' - name: Publish to Conda diff --git a/.github/workflows/release-wave.yml b/.github/workflows/release-wave.yml index f02fe303c9..ddef1d6534 100644 --- a/.github/workflows/release-wave.yml +++ b/.github/workflows/release-wave.yml @@ -51,6 +51,8 @@ jobs: - name: Build Release run: make release env: + # VERSION clashes with conda build. Use PKG_VERSION instead. + PKG_VERSION: ${{ env.VERSION }} NODE_OPTIONS: '--max-old-space-size=8192' - name: Unit Test From a9e3937708dfaa51608580c33aa03564e1612090 Mon Sep 17 00:00:00 2001 From: Marek Mihok Date: Wed, 27 Mar 2024 15:22:09 +0100 Subject: [PATCH 14/14] chore: add python 3.8 --- py/h2o_wave/conda/conda_build_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/py/h2o_wave/conda/conda_build_config.yaml b/py/h2o_wave/conda/conda_build_config.yaml index 5c1b78c223..bfe3c6099c 100644 --- a/py/h2o_wave/conda/conda_build_config.yaml +++ b/py/h2o_wave/conda/conda_build_config.yaml @@ -1,5 +1,6 @@ # TODO: Build a single package for all python versions once this is resolved: https://github.com/conda/conda-build/issues/2611#issuecomment-1239338538 python_version: + - 3.8 - 3.9 - 3.10 - 3.11