Project import generated by Copybara. (#48)

Project import generated by Copybara. (#49) GitOrigin-RevId: 0fdc2918b11abd8900c694506e4dc30d1836d332 Co-authored-by: Snowflake Authors <[email protected]>
snowflakedb · Oct 3, 2023 · 1578599 · 1578599
1 parent 76d191e
commit 1578599
Show file tree

Hide file tree

Showing 340 changed files with 86,419 additions and 1,483 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -1,17 +1,46 @@
-# Make the target platform and the host platform the same
-build --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BUILD_CONDA_ENV=build
-test --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --run_under='//bazel:test_wrapper' --repo_env=BUILD_CONDA_ENV=sf_only
-run --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BUILD_CONDA_ENV=sf_only
-cquery --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BUILD_CONDA_ENV=sf_only
+# Common Default
 
-run:pre_build --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BUILD_CONDA_ENV=build
-build:typecheck --aspects //bazel/mypy:mypy.bzl%mypy_aspect --output_groups=mypy --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --repo_env=BUILD_CONDA_ENV=extended
+# Wrapper to make sure tests are run.
+test --run_under='//bazel:test_wrapper'
 
 # Since integration tests are located in different packages than code under test,
 # the default instrumentation filter would exclude the code under test. This
 # makes bazel consider all the source code in our repo for coverage.
 coverage --instrumentation_filter="-//tests[/:]"
 
-test:extended --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --run_under='//bazel:test_wrapper' --repo_env=BUILD_CONDA_ENV=extended
-run:extended --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --repo_env=BUILD_CONDA_ENV=extended
-cquery:extended --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --repo_env=BUILD_CONDA_ENV=extended
+# Internal definitions
+
+# Make the target platform and the host platform the same
+build:_build --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BAZEL_CONDA_ENV_NAME=build
+build:_sf_only --platforms //bazel/platforms:snowflake_conda_env --host_platform //bazel/platforms:snowflake_conda_env --repo_env=BAZEL_CONDA_ENV_NAME=sf_only
+build:_extended --platforms //bazel/platforms:extended_conda_env --host_platform //bazel/platforms:extended_conda_env --repo_env=BAZEL_CONDA_ENV_NAME=extended
+
+# Public definitions
+
+# Python environment flag, should use in combination with other configs
+
+build:py3.8 --repo_env=BAZEL_CONDA_PYTHON_VERSION=3.8
+build:py3.9 --repo_env=BAZEL_CONDA_PYTHON_VERSION=3.9
+build:py3.10 --repo_env=BAZEL_CONDA_PYTHON_VERSION=3.10
+
+# Default
+
+build --config=_build
+test --config=_sf_only
+run --config=_sf_only
+cquery --config=_sf_only
+
+# Config to sync files
+run:pre_build --config=_build --config=py3.8
+
+# Config to run type check
+build:typecheck --aspects @rules_mypy//:mypy.bzl%mypy_aspect --output_groups=mypy --config=_extended --config=py3.8
+
+# Config to build the doc
+build:docs --config=_sf_only --config=py3.8
+
+# Public the extended setting
+
+test:extended --config=_extended
+run:extended --config=_extended
+cquery:extended --config=_extended
diff --git a/.gitignore b/.gitignore
@@ -85,7 +85,6 @@ dkms.conf
 # User-specific stuff
 .idea/
 
-
 # Gradle and Maven with auto-import
 # When using Gradle or Maven with auto-import, you should exclude module files,
 # since they will be recreated, and may cause churn.  Uncomment if using
@@ -332,8 +331,8 @@ coverage.xml
 # Scrapy stuff:
 .scrapy
 
-# Sphinx documentation
-docs/_build/
+# Auto-generated API reference
+docs/source/api/
 
 # PyBuilder
 target/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,5 @@
 ---
-exclude: ^(.*egg.info.*|docs/.*|.*/parameters.py$|docs/.*|.*\.py_template|.*/experimental/.*)
+exclude: ^(.*egg.info.*|.*/parameters.py$|.*\.py_template|.*/experimental/.*|docs/source/_themes/.*)
 minimum_pre_commit_version: 3.4.0
 repos:
   - repo: https://github.com/asottile/pyupgrade
@@ -26,6 +26,8 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+        exclude_types:
+          - image
         exclude: >
           (?x)^(
               license_header.txt|
@@ -66,12 +68,15 @@ repos:
       - id: buildifier
         args:
           - --warnings=all
+        files: \.(bzl|bazel|sky)$
       # - id: buildifier-lint
       #   args: *args
   - repo: https://github.com/crate-ci/typos  # config: _typos.toml
     rev: v1.16.10
     hooks:
       - id: typos
+        exclude_types:
+          - image
   - repo: https://github.com/lyz-code/yamlfix
     rev: 1.13.0
     hooks:
@@ -89,3 +94,18 @@ repos:
       - id: shellcheck
         args:
           - --severity=warning
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.26.3
+    hooks:
+      - id: check-jsonschema
+        files: ^requirements.*\.yml$
+        args:
+          - --schemafile
+          - bazel/requirements/requirements.schema.json
+      - id: check-github-workflows
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+      - id: python-check-blanket-noqa
+      - id: python-check-blanket-type-ignore
+      - id: python-check-mock-methods
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,14 +1,26 @@
 # Release History
 
-## 1.0.8
+## 1.0.9
 
 ### Behavior Changes
 
+- Model Development: log_loss metric calculation is now distributed.
+
 ### New Features
 
 ### Bug Fixes
 
+- Model Registry: Fix an issue that building images fails with specific docker setup.
+- Model Registry: Fix an issue that unable to embed local ML library when the library is imported by `zipimport`.
+- Model Registry: Fix out-of-date doc about `platform` argument in the `deploy` function.
+- Model Registry: Fix an issue that unable to deploy a GPU-trained PyTorch model to a platform where GPU is not available.
+
+## 1.0.8 (2023-09-15)
+
+### Bug Fixes
+
 - Model Development: Ordinal encoder can be used with mixed input column types.
+- Model Development: Fix an issue when the sklearn default value is `np.nan`.
 - Model Registry: Fix an issue that incorrect docker executable is used when building images.
 - Model Registry: Fix an issue that specifying `token` argument when using
 `snowflake.ml.model.models.huggingface_pipeline.HuggingFacePipelineModel` with `transformers < 4.32.0` is not effective.

diff --git a/WORKSPACE b/WORKSPACE
@@ -47,6 +47,16 @@ aspect_bazel_lib_dependencies()
 
 register_yq_toolchains()
 
+local_repository(
+    name = "rules_sphinx",
+    path = "./third_party/rules_sphinx",
+)
+
+local_repository(
+    name = "rules_mypy",
+    path = "./third_party/rules_mypy",
+)
+
 # Below two conda environments (toolchains) are created and they require different
 # constraint values. Two platforms defined in bazel/platforms/BUILD provide those
 # constraint values. A toolchain matches a platform as long as the platform provides
@@ -63,26 +73,29 @@ load("@SnowML//bazel/environments:fetch_conda_env_config.bzl", "fetch_conda_env_
 
 fetch_conda_env_config(name = "fetch_conda_env_config_repo")
 
-load("@fetch_conda_env_config_repo//:config.bzl", "COMPATIBLE_TARGET", "ENVIRONMENT", "NAME")
+load("@fetch_conda_env_config_repo//:config.bzl", "COMPATIBLE_TARGET", "ENVIRONMENT", "NAME", "PYTHON_VERSION")
+
+PYTHON_VERSION_DISPLAY_NAME = "".join(PYTHON_VERSION.split("."))
 
 load_conda(
-    conda_repo_name = "{}_conda".format(NAME),
+    conda_repo_name = "{}_conda_{}".format(NAME, PYTHON_VERSION_DISPLAY_NAME),
     quiet = True,
 )
 
 conda_create(
-    name = "{}_env".format(NAME),
+    name = "{}_env_{}".format(NAME, PYTHON_VERSION_DISPLAY_NAME),
     timeout = 3600,
     clean = False,
-    conda_repo_name = "{}_conda".format(NAME),
+    conda_repo_name = "{}_conda_{}".format(NAME, PYTHON_VERSION_DISPLAY_NAME),
     coverage_tool = "@//bazel/coverage_tool:coverage_tool.py",
     environment = ENVIRONMENT,
+    python_version = PYTHON_VERSION,
     quiet = True,
 )
 
 register_toolchain(
-    name = "{}_env_repo".format(NAME),
-    env = "{}_env".format(NAME),
+    name = "{}_env_{}_repo".format(NAME, PYTHON_VERSION_DISPLAY_NAME),
+    env = "{}_env_{}".format(NAME, PYTHON_VERSION_DISPLAY_NAME),
     target_compatible_with = COMPATIBLE_TARGET,
-    toolchain_name = "py3_toolchain_{}_env".format(NAME),
+    toolchain_name = "py{}_toolchain_{}_env".format(PYTHON_VERSION_DISPLAY_NAME, NAME),
 )
diff --git a/bazel/environments/conda-env-build.yml b/bazel/environments/conda-env-build.yml
@@ -7,13 +7,14 @@ channels:
   - nodefaults
 dependencies:
   - absl-py==1.3.0
-  - conda-libmamba-solver==23.3.0
+  - conda-libmamba-solver==23.7.0
   - inflection==0.5.1
   - jsonschema==3.2.0
   - lightgbm==3.3.5
   - numpy==1.24.3
   - packaging==23.0
-  - python==3.8.13
+  - pytimeparse==1.1.8
   - ruamel.yaml==0.17.21
   - scikit-learn==1.3.0
+  - sphinx==5.0.2
   - xgboost==1.7.3
diff --git a/bazel/environments/conda-env-snowflake.yml b/bazel/environments/conda-env-snowflake.yml
@@ -12,7 +12,7 @@ dependencies:
   - boto3==1.24.28
   - cachetools==4.2.2
   - cloudpickle==2.0.0
-  - conda-libmamba-solver==23.3.0
+  - conda-libmamba-solver==23.7.0
   - coverage==6.3.2
   - cryptography==39.0.1
   - flask-cors==3.0.10
@@ -31,7 +31,7 @@ dependencies:
   - pandas==1.5.3
   - protobuf==3.20.3
   - pytest==7.1.2
-  - python==3.8.13
+  - pytimeparse==1.1.8
   - pytorch==2.0.1
   - pyyaml==6.0
   - requests==2.29.0
@@ -40,9 +40,11 @@ dependencies:
   - scikit-learn==1.3.0
   - scipy==1.9.3
   - sentencepiece==0.1.99
-  - snowflake-connector-python==3.1.0
-  - snowflake-snowpark-python==1.5.1
-  - sqlparse==0.4.3
+  - shap==0.42.1
+  - snowflake-connector-python==3.2.0
+  - snowflake-snowpark-python==1.6.1
+  - sphinx==5.0.2
+  - sqlparse==0.4.4
   - tensorflow==2.10.0
   - torchdata==0.6.1
   - transformers==4.29.2

diff --git a/bazel/environments/conda-env.yml b/bazel/environments/conda-env.yml
@@ -17,7 +17,7 @@ dependencies:
   - conda-forge::starlette==0.27.0
   - conda-forge::types-PyYAML==6.0.12
   - conda-forge::types-cachetools==4.2.2
-  - conda-libmamba-solver==23.3.0
+  - conda-libmamba-solver==23.7.0
   - coverage==6.3.2
   - cryptography==39.0.1
   - flask-cors==3.0.10
@@ -36,7 +36,7 @@ dependencies:
   - pandas==1.5.3
   - protobuf==3.20.3
   - pytest==7.1.2
-  - python==3.8.13
+  - pytimeparse==1.1.8
   - pytorch==2.0.1
   - pyyaml==6.0
   - requests==2.29.0
@@ -45,9 +45,11 @@ dependencies:
   - scikit-learn==1.3.0
   - scipy==1.9.3
   - sentencepiece==0.1.99
-  - snowflake-connector-python==3.1.0
-  - snowflake-snowpark-python==1.5.1
-  - sqlparse==0.4.3
+  - shap==0.42.1
+  - snowflake-connector-python==3.2.0
+  - snowflake-snowpark-python==1.6.1
+  - sphinx==5.0.2
+  - sqlparse==0.4.4
   - tensorflow==2.10.0
   - torchdata==0.6.1
   - transformers==4.29.2

diff --git a/bazel/environments/fetch_conda_env_config.bzl b/bazel/environments/fetch_conda_env_config.bzl
@@ -1,6 +1,7 @@
 def _fetch_conda_env_config_impl(rctx):
     # read the particular environment variable we are interested in
-    config = rctx.os.environ.get("BUILD_CONDA_ENV", "extended").lower()
+    env_name = rctx.os.environ.get("BAZEL_CONDA_ENV_NAME", "extended").lower()
+    python_ver = rctx.os.environ.get("BAZEL_CONDA_PYTHON_VERSION", "3.8").lower()
 
     # necessary to create empty BUILD file for this rule
     # which will be located somewhere in the Bazel build files
@@ -21,8 +22,8 @@ def _fetch_conda_env_config_impl(rctx):
         },
     }
 
-    if config not in conda_env_map.keys():
-        fail("Unsupported conda env {} specified. Only {} is supported.".format(config, repr(conda_env_map.keys())))
+    if env_name not in conda_env_map.keys():
+        fail("Unsupported conda env {} specified. Only {} is supported.".format(env_name, repr(conda_env_map.keys())))
 
     # create a temporary file called config.bzl to be loaded into WORKSPACE
     # passing in any desired information from this rule implementation
@@ -32,10 +33,11 @@ def _fetch_conda_env_config_impl(rctx):
 NAME = {}
 ENVIRONMENT = {}
 COMPATIBLE_TARGET = {}
-""".format(repr(config), repr(conda_env_map[config]["environment"]), repr(conda_env_map[config]["compatible_target"])),
+PYTHON_VERSION = {}
+""".format(repr(env_name), repr(conda_env_map[env_name]["environment"]), repr(conda_env_map[env_name]["compatible_target"]), repr(python_ver)),
     )
 
 fetch_conda_env_config = repository_rule(
     implementation = _fetch_conda_env_config_impl,
-    environ = ["BUILD_CONDA_ENV"],
+    environ = ["BAZEL_CONDA_ENV_NAME", "BAZEL_CONDA_PYTHON_VERSION"],
 )
diff --git a/bazel/get_affected_targets.sh b/bazel/get_affected_targets.sh
@@ -17,6 +17,8 @@
 
 set -o pipefail
 set -u
+set -e
+
 PROG=$0
 
 help() {
@@ -79,6 +81,7 @@ SeedFileContent
 "${bazel}" run --config=pre_build :bazel-diff --script_path="${bazel_diff}"
 
 git -C "${workspace_path}" checkout "${pr_revision}" --quiet
+trap 'git -C "${workspace_path}" checkout "${current_revision}" --quiet' EXIT
 
 echo "Generating Hashes for Revision '${pr_revision}'"
 
@@ -109,5 +112,3 @@ EndOfMessage
 # -- End of Query Rules Heredoc --
 
 "${bazel}" query --query_file="${filter_query_rules_file}" >"${output_path}"
-
-git -C "${workspace_path}" checkout "${current_revision}" --quiet