Release snowflake-ml-python: 1.0.12

Co-authored-by: Snowflake Authors <[email protected]>

.bazelrc

@@ -15,6 +15,8 @@ build:_build --platforms //bazel/platforms:snowflake_conda_env --host_platform /
 build:_sf_only --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BAZEL_CONDA_ENV_NAME=sf_only
 build:_extended --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --repo_env=BAZEL_CONDA_ENV_NAME=extended
 build:_extended_oss --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --repo_env=BAZEL_CONDA_ENV_NAME=extended_oss
+build:_extended_gpu_oss --platforms //bazel/platforms:extended_conda_gpu_env --host_platform //bazel/platforms:extended_conda_gpu_env --repo_env=BAZEL_CONDA_ENV_NAME=extended_gpu_oss
+
 
 # Public definitions
 
@@ -37,6 +39,7 @@ run:pre_build --config=_build --config=py3.8
 # Config to run type check
 build:typecheck --aspects @rules_mypy//:mypy.bzl%mypy_aspect --output_groups=mypy --config=_extended --config=py3.8
 build:typecheck_oss --aspects @rules_mypy//:mypy.bzl%mypy_aspect --output_groups=mypy --config=_extended_oss --config=py3.8
+build:typecheck_gpu_oss --aspects @rules_mypy//:mypy.bzl%mypy_aspect --output_groups=mypy --config=_extended_gpu_oss --config=py3.8
 
 # Config to build the doc
 build:docs --config=_sf_only --config=py3.8
@@ -49,3 +52,6 @@ cquery:extended --config=_extended
 test:extended_oss --config=_extended_oss
 run:extended_oss --config=_extended_oss
 cquery:extended_oss --config=_extended_oss
+test:extended_gpu_oss --config=_extended_gpu_oss
+run:extended_gpu_oss --config=_extended_gpu_oss
+cquery:extended_gpu_oss --config=_extended_gpu_oss

.pre-commit-config.yaml

@@ -62,13 +62,18 @@ repos:
     rev: v0.36.0
     hooks:
       - id: markdownlint-fix
+        language_version: 16.20.2
   - repo: https://github.com/keith/pre-commit-buildifier
     rev: 6.0.0
     hooks:
       - id: buildifier
         args:
           - --warnings=all
         files: \.(bzl|bazel|sky)$
+        exclude: >
+          (?x)^(
+              snowflake/ml/requirements.bzl
+          )$
       # - id: buildifier-lint
       #   args: *args
   - repo: https://github.com/crate-ci/typos  # config: _typos.toml

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Release History
 
+## 1.0.12
+
+### Bug Fixes
+
+- Model Registry: Fix regression issue that container logging is not shown during model deployment to SPCS.
+- Model Development: Enhance the column capacity of OrdinalEncoder.
+- Model Registry: Fix unbound `batch_size`` error when deploying a model other than Hugging Face Pipeline
+   and LLM with GPU on SPCS.
+
+### Behavior Changes
+
+- Model Registry: Raise early error when deploying to SPCS with db/schema that starts with underscore.
+- Model Registry: `conda-forge` channel is now automatically added to channel lists when deploying to SPCS.
+- Model Registry: `relax_version` will not strip all version specifier, instead it will relax `==x.y.z` specifier to
+  `>=x.y,<(x+1)`.
+- Model Registry: Python with different patchlevel but the same major and minor will not result a warning when loading
+  the model via Model Registry and would be considered to use when deploying to SPCS.
+- Model Registry: When logging a `snowflake.ml.model.models.huggingface_pipeline.HuggingFacePipelineModel` object,
+  versions of local installed libraries won't be picked as dependencies of models, instead it will pick up some pre-
+  defined dependencies to improve user experience.
+
+### New Features
+
+- Model Registry: Enable best-effort SPCS job/service log streaming when logging level is set to INFO.
+
 ## 1.0.11
 
 ### New Features
@@ -17,6 +42,7 @@
 - Model Development: Fix support for XGBoost and LightGBM models using SKLearn Grid Search and Randomized Search model selectors.
 - Model Development: DecimalType is now supported as a DataType.
 - Model Development: Fix metrics compatibility with Snowpark Dataframes that use Snowflake identifiers
+- Model Registry: Resolve 'delete_deployment' not deleting the SPCS service in certain cases.
 
 ## 1.0.10
 

WORKSPACE

@@ -1,6 +1,5 @@
 workspace(name = "SnowML")
 
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar")
 
 http_jar(
@@ -24,14 +23,19 @@ load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 
 bazel_skylib_workspace()
 
-# Latest @ 2023-06-20
-# Replace with released version once newer version released.
-git_repository(
+http_archive(
     name = "rules_python",
-    commit = "0d59fcf561f6d2c4705924bc17c151fb4b998841",
-    remote = "https://github.com/bazelbuild/rules_python.git",
+    patch_args = ["-p1"],
+    patches = ["@//third_party/rules_python:packaging.patch"],
+    sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
+    strip_prefix = "rules_python-0.26.0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz",
 )
 
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
 load("//third_party/rules_conda:defs.bzl", "conda_create", "load_conda", "register_toolchain")
 
 http_archive(

bazel/environments/BUILD.bazel

@@ -2,4 +2,5 @@ exports_files([
     "conda-env-snowflake.yml",
     "conda-env-build.yml",
     "conda-env.yml",
+    "conda-gpu-env.yml",
 ])

bazel/environments/conda-env-snowflake.yml

@@ -19,6 +19,7 @@ dependencies:
   - flask==2.1.3
   - fsspec==2023.3.0
   - httpx==0.23.0
+  - importlib_resources==5.1.4
   - inflection==0.5.1
   - joblib==1.1.1
   - jsonschema==3.2.0

bazel/environments/conda-env.yml

@@ -24,6 +24,7 @@ dependencies:
   - flask==2.1.3
   - fsspec==2023.3.0
   - httpx==0.23.0
+  - importlib_resources==5.1.4
   - inflection==0.5.1
   - joblib==1.1.1
   - jsonschema==3.2.0
@@ -62,3 +63,4 @@ dependencies:
   - pip:
       - --extra-index-url https://pypi.org/simple
       - peft==0.5.0
+      - vllm==0.2.1.post1

bazel/environments/conda-gpu-env.yml

@@ -0,0 +1,68 @@
+---
+# DO NOT EDIT!
+# Generate by running 'bazel run --config=pre_build //bazel/requirements:sync_requirements'
+
+channels:
+  - https://repo.anaconda.com/pkgs/snowflake
+  - nodefaults
+dependencies:
+  - absl-py==1.3.0
+  - aiohttp==3.8.3
+  - anyio==3.5.0
+  - boto3==1.24.28
+  - cachetools==4.2.2
+  - cloudpickle==2.0.0
+  - conda-forge::accelerate==0.22.0
+  - conda-forge::mypy==1.5.1
+  - conda-forge::starlette==0.27.0
+  - conda-forge::types-PyYAML==6.0.12
+  - conda-forge::types-cachetools==4.2.2
+  - conda-libmamba-solver==23.7.0
+  - coverage==6.3.2
+  - cryptography==39.0.1
+  - flask-cors==3.0.10
+  - flask==2.1.3
+  - fsspec==2023.3.0
+  - httpx==0.23.0
+  - importlib_resources==5.1.4
+  - inflection==0.5.1
+  - joblib==1.1.1
+  - jsonschema==3.2.0
+  - lightgbm==3.3.5
+  - mlflow==2.3.1
+  - moto==4.0.11
+  - networkx==2.8.4
+  - numpy==1.24.3
+  - nvidia::cuda==11.7.*
+  - packaging==23.0
+  - pandas==1.5.3
+  - protobuf==3.20.3
+  - pytest==7.4.0
+  - pytimeparse==1.1.8
+  - pytorch::pytorch-cuda==11.7.*
+  - pytorch::pytorch==2.0.1
+  - pyyaml==6.0
+  - requests==2.29.0
+  - ruamel.yaml==0.17.21
+  - s3fs==2023.3.0
+  - scikit-learn==1.3.0
+  - scipy==1.9.3
+  - sentencepiece==0.1.99
+  - shap==0.42.1
+  - snowflake-connector-python==3.2.0
+  - snowflake-snowpark-python==1.6.1
+  - sphinx==5.0.2
+  - sqlparse==0.4.4
+  - tensorflow==2.10.0
+  - tokenizers==0.13.2
+  - torchdata==0.6.1
+  - transformers==4.32.1
+  - types-protobuf==4.23.0.1
+  - types-requests==2.30.0.0
+  - typing-extensions==4.5.0
+  - xgboost==1.7.3
+  - pip
+  - pip:
+      - --extra-index-url https://pypi.org/simple
+      - peft==0.5.0
+      - vllm==0.2.1.post1

bazel/environments/fetch_conda_env_config.bzl

@@ -16,6 +16,10 @@ def _fetch_conda_env_config_impl(rctx):
             "compatible_target": ["@SnowML//bazel/platforms:extended_conda_channels"],
             "environment": "@//bazel/environments:conda-env.yml",
         },
+        "extended_gpu_oss": {
+            "compatible_target": ["@SnowML//bazel/platforms:extended_conda_channels", "@SnowML//bazel/platforms:has_gpu"],
+            "environment": "@//bazel/environments:conda-gpu-env.yml",
+        },
         # `extended_oss` is the extended env for OSS repo which is a  strict subset of `extended`.
         # It's intended for development without dev VPN.
         "extended_oss": {

bazel/platforms/BUILD.bazel

@@ -2,6 +2,8 @@ package(default_visibility = ["//visibility:public"])
 
 constraint_setting(name = "conda_env")
 
+constraint_setting(name = "gpu_env")
+
 constraint_value(
     name = "snowflake_conda_channel",
     constraint_setting = ":conda_env",
@@ -12,14 +14,39 @@ constraint_value(
     constraint_setting = ":conda_env",
 )
 
+constraint_value(
+    name = "has_gpu",
+    constraint_setting = ":gpu_env",
+)
+
+constraint_value(
+    name = "no_gpu",
+    constraint_setting = ":gpu_env",
+)
+
 platform(
     name = "snowflake_conda_env",
-    constraint_values = [":snowflake_conda_channel"],
+    constraint_values = [
+        ":snowflake_conda_channel",
+        ":no_gpu",
+    ],
     parents = ["@local_config_platform//:host"],
 )
 
 platform(
     name = "extended_conda_env",
-    constraint_values = [":extended_conda_channels"],
+    constraint_values = [
+        ":extended_conda_channels",
+        ":no_gpu",
+    ],
+    parents = ["@local_config_platform//:host"],
+)
+
+platform(
+    name = "extended_conda_gpu_env",
+    constraint_values = [
+        ":extended_conda_channels",
+        ":has_gpu",
+    ],
     parents = ["@local_config_platform//:host"],
 )

bazel/py_rules.bzl

@@ -18,6 +18,9 @@ Overriding default implementation of py_{binary|library|test} to add additional
      //bazel/platforms:snowflake_conda_env
    - None of the target can be built with the host platform (@local_config_platform//:host). However
      that platform is not the default platform (see .bazelrc).
+5. Similarly, a boolean attribute "require_gpu" is available in all the wrapped rules.
+   The value of this flag affects which platform and toolchain could be running the target as above.
+   Only toolchain supporting GPU would be able to run target tagged with "require_gpu=True"
 
 ### Setup
 ```python
@@ -44,33 +47,52 @@ def py_genrule(**attrs):
 
 _COMPATIBLE_WITH_SNOWPARK_TAG = "wheel_compatible_with_snowpark"
 
-def _add_target_compatibility_labels(compatible_with_snowpark, attrs):
+def _add_target_compatibility_labels(compatible_with_snowpark, require_gpu, attrs):
+    if compatible_with_snowpark and require_gpu:
+        fail("`require_gpu` is not compatible with snowpark.!")
     if compatible_with_snowpark:
         attrs["target_compatible_with"] = select({
             "//bazel/platforms:extended_conda_channels": [],
             "//bazel/platforms:snowflake_conda_channel": [],
             "//conditions:default": ["@platforms//:incompatible"],
+        }) + select({
+            "//bazel/platforms:has_gpu": [],
+            "//bazel/platforms:no_gpu": [],
+            "//conditions:default": ["@platforms//:incompatible"],
+        })
+    elif require_gpu:
+        attrs["target_compatible_with"] = select({
+            "//bazel/platforms:extended_conda_channels": [],
+            "//conditions:default": ["@platforms//:incompatible"],
+        }) + select({
+            "//bazel/platforms:has_gpu": [],
+            "//conditions:default": ["@platforms//:incompatible"],
         })
     else:
         attrs["target_compatible_with"] = select({
             "//bazel/platforms:extended_conda_channels": [],
             "//conditions:default": ["@platforms//:incompatible"],
+        }) + select({
+            "//bazel/platforms:has_gpu": [],
+            "//bazel/platforms:no_gpu": [],
+            "//conditions:default": ["@platforms//:incompatible"],
         })
 
-def py_binary(compatible_with_snowpark = True, **attrs):
+def py_binary(compatible_with_snowpark = True, require_gpu = False, **attrs):
     """Modified version of core py_binary to add check for experimental dependencies.
 
     See the Bazel core [py_binary](https://docs.bazel.build/versions/master/be/python.html#py_binary) documentation.
 
     Args:
       compatible_with_snowpark: see file-level document.
+      require_gpu: see file-level document.
       **attrs: Rule attributes
     """
     if not check_for_tests_dependencies(native.package_name(), attrs):
         fail("A target in src cannot depend on packages in tests!")
     if not check_for_experimental_dependencies(native.package_name(), attrs):
         fail("Non Experimental Target cannot depend on experimental library!")
-    _add_target_compatibility_labels(compatible_with_snowpark, attrs)
+    _add_target_compatibility_labels(compatible_with_snowpark, require_gpu, attrs)
 
     # Disable bazel's behavior to add __init__.py files to modules by default. This causes import errors. Context:
     # * https://bazel.build/reference/be/python#py_test.legacy_create_init
@@ -82,7 +104,7 @@ def py_binary(compatible_with_snowpark = True, **attrs):
     })
     native_py_binary(**attrs)
 
-def py_library(compatible_with_snowpark = True, **attrs):
+def py_library(compatible_with_snowpark = True, require_gpu = False, **attrs):
     """Modified version of core py_library to add additional imports and check for experimental dependencies.
 
     See the Bazel core [py_library](https://docs.bazel.build/versions/master/be/python.html#py_library) documentation.
@@ -103,23 +125,25 @@ def py_library(compatible_with_snowpark = True, **attrs):
 
     Args:
       compatible_with_snowpark: see file-level document.
+      require_gpu: see file-level document.
       **attrs: Rule attributes
     """
     if not check_for_tests_dependencies(native.package_name(), attrs):
         fail("A target in src cannot depend on packages in tests!")
     if not check_for_experimental_dependencies(native.package_name(), attrs):
         fail("Non Experimental Target cannot depend on experimental library!")
-    _add_target_compatibility_labels(compatible_with_snowpark, attrs)
+    _add_target_compatibility_labels(compatible_with_snowpark, require_gpu, attrs)
 
     native_py_library(**attrs)
 
-def py_test(compatible_with_snowpark = True, **attrs):
+def py_test(compatible_with_snowpark = True, require_gpu = False, **attrs):
     """Modified version of core py_binary to add check for experimental dependencies.
 
     See the Bazel core [py_test](https://docs.bazel.build/versions/master/be/python.html#py_test) documentation.
 
     Args:
       compatible_with_snowpark: see file-level document.
+      require_gpu: see file-level document.
       **attrs: Rule attributes
     """
     if not check_for_test_name(native.package_name(), attrs):
@@ -128,7 +152,7 @@ def py_test(compatible_with_snowpark = True, **attrs):
         fail("A target in src cannot depend on packages in tests!")
     if not check_for_experimental_dependencies(native.package_name(), attrs):
         fail("Non Experimental Target cannot depend on experimental library!")
-    _add_target_compatibility_labels(compatible_with_snowpark, attrs)
+    _add_target_compatibility_labels(compatible_with_snowpark, require_gpu, attrs)
 
     # Disable bazel's behavior to add __init__.py files to modules by default. This causes import errors. Context:
     # * https://bazel.build/reference/be/python#py_test.legacy_create_init

bazel/requirements/BUILD.bazel

@@ -38,6 +38,11 @@ _GENERATED_REQUIREMENTS_FILES = {
         "generated": "conda-env.yml",
         "target": "//bazel/environments:conda-env.yml",
     },
+    "conda_gpu_env_yml": {
+        "cmd": "--mode dev_gpu_version --format conda_env",
+        "generated": "conda-gpu-env.yml",
+        "target": "//bazel/environments:conda-gpu-env.yml",
+    },
     "conda_meta": {
         "cmd": "--mode version_requirements --format conda_meta --version " + VERSION,
         "generated": "meta.yaml",