Project import generated by Copybara. (#93)

GitOrigin-RevId: 1c075f3455c3d9b57e2eba985bfedaaa770bb1bf

Co-authored-by: Snowflake Authors <[email protected]>

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 1.3.1
+
+### New Features
+
+- FileSet: `snowflake.ml.fileset.sfcfs.SFFileSystem` can now be used in UDFs and stored procedures.
+
 ## 1.3.0
 
 ### Bug Fixes

bazel/environments/BUILD.bazel

@@ -3,4 +3,5 @@ exports_files([
     "conda-env-build.yml",
     "conda-env.yml",
     "conda-gpu-env.yml",
+    "conda-env-build-test.yml",
 ])

bazel/environments/conda-env-build-test.yml

@@ -0,0 +1,12 @@
+---
+# DO NOT EDIT!
+# Generate by running 'bazel run --config=pre_build //bazel/requirements:sync_requirements'
+
+channels:
+  - https://repo.anaconda.com/pkgs/snowflake
+  - nodefaults
+dependencies:
+  - inflection==0.5.1
+  - psutil==5.9.0
+  - pytest-xdist==3.5.0
+  - pytest==7.4.0

bazel/environments/conda-env-build.yml

@@ -12,7 +12,7 @@ dependencies:
   - inflection==0.5.1
   - jsonschema==3.2.0
   - lightgbm==3.3.5
-  - numpy==1.24.3
+  - numpy==1.23.5
   - packaging==23.0
   - ruamel.yaml==0.17.21
   - scikit-learn==1.3.0

bazel/environments/conda-env-snowflake.yml

@@ -12,27 +12,29 @@ dependencies:
   - boto3==1.24.28
   - build==0.10.0
   - cachetools==4.2.2
-  - cloudpickle==2.0.0
+  - cloudpickle==2.2.1
   - conda-libmamba-solver==23.7.0
   - coverage==6.3.2
   - cryptography==39.0.1
   - flask-cors==3.0.10
   - flask==2.1.3
   - fsspec==2023.3.0
   - httpx==0.23.0
-  - importlib_resources==5.1.4
+  - importlib_resources==6.1.1
   - inflection==0.5.1
   - joblib==1.1.1
   - jsonschema==3.2.0
   - lightgbm==3.3.5
   - mlflow==2.3.1
   - moto==4.0.11
   - networkx==2.8.4
-  - numpy==1.24.3
+  - numpy==1.23.5
   - packaging==23.0
   - pandas==1.5.3
   - protobuf==3.20.3
+  - psutil==5.9.0
   - pyarrow==10.0.1
+  - pytest-xdist==3.5.0
   - pytest==7.4.0
   - pytimeparse==1.1.8
   - pytorch==2.0.1
@@ -43,13 +45,14 @@ dependencies:
   - s3fs==2023.3.0
   - scikit-learn==1.3.0
   - scipy==1.9.3
+  - sentence-transformers==2.2.2
   - sentencepiece==0.1.99
   - shap==0.42.1
-  - snowflake-connector-python==3.2.0
-  - snowflake-snowpark-python==1.8.0
+  - snowflake-connector-python==3.5.0
+  - snowflake-snowpark-python==1.11.1
   - sphinx==5.0.2
   - sqlparse==0.4.4
-  - tensorflow==2.10.0
+  - tensorflow==2.12.0
   - tokenizers==0.13.2
   - toml==0.10.2
   - torchdata==0.6.1

bazel/environments/conda-env.yml

@@ -12,7 +12,7 @@ dependencies:
   - boto3==1.24.28
   - build==0.10.0
   - cachetools==4.2.2
-  - cloudpickle==2.0.0
+  - cloudpickle==2.2.1
   - conda-forge::accelerate==0.22.0
   - conda-forge::mypy==1.5.1
   - conda-forge::starlette==0.27.0
@@ -25,19 +25,21 @@ dependencies:
   - flask==2.1.3
   - fsspec==2023.3.0
   - httpx==0.23.0
-  - importlib_resources==5.1.4
+  - importlib_resources==6.1.1
   - inflection==0.5.1
   - joblib==1.1.1
   - jsonschema==3.2.0
   - lightgbm==3.3.5
   - mlflow==2.3.1
   - moto==4.0.11
   - networkx==2.8.4
-  - numpy==1.24.3
+  - numpy==1.23.5
   - packaging==23.0
   - pandas==1.5.3
   - protobuf==3.20.3
+  - psutil==5.9.0
   - pyarrow==10.0.1
+  - pytest-xdist==3.5.0
   - pytest==7.4.0
   - pytimeparse==1.1.8
   - pytorch==2.0.1
@@ -48,13 +50,14 @@ dependencies:
   - s3fs==2023.3.0
   - scikit-learn==1.3.0
   - scipy==1.9.3
+  - sentence-transformers==2.2.2
   - sentencepiece==0.1.99
   - shap==0.42.1
-  - snowflake-connector-python==3.2.0
-  - snowflake-snowpark-python==1.8.0
+  - snowflake-connector-python==3.5.0
+  - snowflake-snowpark-python==1.11.1
   - sphinx==5.0.2
   - sqlparse==0.4.4
-  - tensorflow==2.10.0
+  - tensorflow==2.12.0
   - tokenizers==0.13.2
   - toml==0.10.2
   - torchdata==0.6.1

bazel/environments/conda-gpu-env.yml

@@ -12,7 +12,7 @@ dependencies:
   - boto3==1.24.28
   - build==0.10.0
   - cachetools==4.2.2
-  - cloudpickle==2.0.0
+  - cloudpickle==2.2.1
   - conda-forge::accelerate==0.22.0
   - conda-forge::mypy==1.5.1
   - conda-forge::starlette==0.27.0
@@ -25,20 +25,22 @@ dependencies:
   - flask==2.1.3
   - fsspec==2023.3.0
   - httpx==0.23.0
-  - importlib_resources==5.1.4
+  - importlib_resources==6.1.1
   - inflection==0.5.1
   - joblib==1.1.1
   - jsonschema==3.2.0
   - lightgbm==3.3.5
   - mlflow==2.3.1
   - moto==4.0.11
   - networkx==2.8.4
-  - numpy==1.24.3
+  - numpy==1.23.5
   - nvidia::cuda==11.7.*
   - packaging==23.0
   - pandas==1.5.3
   - protobuf==3.20.3
+  - psutil==5.9.0
   - pyarrow==10.0.1
+  - pytest-xdist==3.5.0
   - pytest==7.4.0
   - pytimeparse==1.1.8
   - pytorch::pytorch-cuda==11.7.*
@@ -50,13 +52,14 @@ dependencies:
   - s3fs==2023.3.0
   - scikit-learn==1.3.0
   - scipy==1.9.3
+  - sentence-transformers==2.2.2
   - sentencepiece==0.1.99
   - shap==0.42.1
-  - snowflake-connector-python==3.2.0
-  - snowflake-snowpark-python==1.8.0
+  - snowflake-connector-python==3.5.0
+  - snowflake-snowpark-python==1.11.1
   - sphinx==5.0.2
   - sqlparse==0.4.4
-  - tensorflow==2.10.0
+  - tensorflow==2.12.0
   - tokenizers==0.13.2
   - toml==0.10.2
   - torchdata==0.6.1

bazel/requirements/BUILD.bazel

@@ -23,6 +23,11 @@ _SYNC_BAZEL_CMD = "bazel run --config=pre_build //{current_path}:{sync_rule}".fo
 )
 
 _GENERATED_REQUIREMENTS_FILES = {
+    "conda_env_build_test_yml": {
+        "cmd": "--mode dev_version --format conda_env --filter_by_tag build_test_env",
+        "generated": "conda-env-build-test.yml",
+        "target": "//bazel/environments:conda-env-build-test.yml",
+    },
     "conda_env_build_yml": {
         "cmd": "--mode dev_version --format conda_env --filter_by_tag build_essential",
         "generated": "conda-env-build.yml",

bazel/requirements/requirements.schema.json

@@ -98,7 +98,8 @@
                 "udf_inference",
                 "spcs_inference",
                 "model_packaging",
-                "build_essential"
+                "build_essential",
+                "build_test_env"
               ],
               "type": "string"
             }

ci/build_and_run_tests.sh

@@ -39,7 +39,6 @@ ENV="pip"
 WITH_SNOWPARK=false
 MODE="continuous_run"
 PYTHON_VERSION=3.8
-PYTHON_JENKINS_ENABLE="/opt/rh/rh-python38/enable"
 PYTHON_ENABLE_SCRIPT="bin/activate"
 SNOWML_DIR="snowml"
 SNOWPARK_DIR="snowpark-python"
@@ -151,48 +150,30 @@ case ${PYTHON_VERSION} in
     else
         PYTHON_EXECUTABLE="python3.8"
     fi
-    PYTHON_JENKINS_ENABLE="/opt/rh/rh-python38/enable"
     ;;
   3.9)
     if [ ${IS_NT} = true ]; then
         PYTHON_EXECUTABLE="py -3.9"
     else
         PYTHON_EXECUTABLE="python3.9"
     fi
-    PYTHON_JENKINS_ENABLE="/opt/rh/rh-python39/enable"
     ;;
   3.10)
     if [ ${IS_NT} = true ]; then
         PYTHON_EXECUTABLE="py -3.10"
     else
         PYTHON_EXECUTABLE="python3.10"
     fi
-    PYTHON_JENKINS_ENABLE="/opt/rh/rh-python310/enable"
     ;;
   3.11)
     if [ ${IS_NT} = true ]; then
         PYTHON_EXECUTABLE="py -3.11"
     else
         PYTHON_EXECUTABLE="python3.11"
     fi
-    PYTHON_JENKINS_ENABLE="/opt/rh/rh-python311/enable"
     ;;
 esac
 
-# TODO(SNOW-901629): Use native python provided in the node once SNOW-1046060 resolved
-if [[ "${ENV}" = "pip" && ${IS_NT} = false ]]; then
-    set +eu
-    # shellcheck source=/dev/null
-    source ${PYTHON_JENKINS_ENABLE}
-    PYTHON_EXIST=$?
-    if [ $PYTHON_EXIST -ne 0 ]; then
-        echo "Failed to execute tests: ${PYTHON_EXECUTABLE} is not installed."
-        rm -rf "${TEMP_TEST_DIR}"
-        exit ${PYTHON_EXIST}
-    fi
-    set -eu
-fi
-
 cd "${WORKSPACE}"
 
 # Check and download yq if not presented.
@@ -311,7 +292,7 @@ if [ "${ENV}" = "pip" ]; then
     python --version
     python -m pip install --upgrade pip
     python -m pip list
-    python -m pip install "snowflake_ml_python-${VERSION}-py3-none-any.whl[all]" "pytest-xdist[psutil]==2.5.0" -r "${WORKSPACE}/${SNOWML_DIR}/requirements.txt" --no-cache-dir --force-reinstall
+    python -m pip install "snowflake_ml_python-${VERSION}-py3-none-any.whl[all]" -r "${WORKSPACE}/${SNOWML_DIR}/requirements.txt" --no-cache-dir --force-reinstall
     if [ "${WITH_SNOWPARK}" = true ]; then
         cp "$(find "${WORKSPACE}" -maxdepth 1 -iname 'snowflake_snowpark_python-*.whl')" "${TEMP_TEST_DIR}"
         python -m pip install "$(find . -maxdepth 1 -iname 'snowflake_snowpark_python-*.whl')" --no-deps --force-reinstall
@@ -331,7 +312,8 @@ else
     conda clean --all --force-pkgs-dirs -y
 
     # Create testing env
-    conda create -y -p testenv -c "${WORKSPACE}/conda-bld" -c "https://repo.anaconda.com/pkgs/snowflake/" --override-channels "python=${PYTHON_VERSION}" snowflake-ml-python "py==1.9.0" "pytest-xdist==2.5.0" psutil inflection "${OPTIONAL_REQUIREMENTS[@]}"
+    conda create -y -p testenv -c "${WORKSPACE}/conda-bld" -c "https://repo.anaconda.com/pkgs/snowflake/" --override-channels "python=${PYTHON_VERSION}" snowflake-ml-python "${OPTIONAL_REQUIREMENTS[@]}"
+    conda env update -p testenv -f "${WORKSPACE}/${SNOWML_DIR}/bazel/environments/conda-env-build-test.yml"
     conda list -p testenv
 
     # Run integration tests

ci/conda_recipe/meta.yaml

@@ -17,7 +17,7 @@ build:
   noarch: python
 package:
   name: snowflake-ml-python
-  version: 1.3.0
+  version: 1.3.1
 requirements:
   build:
     - python
@@ -29,7 +29,7 @@ requirements:
     - cachetools>=3.1.1,<6
     - cloudpickle>=2.0.0
     - fsspec>=2022.11,<2024
-    - importlib_resources>=5.1.4, <6
+    - importlib_resources>=6.1.1, <7
     - numpy>=1.23,<2
     - packaging>=20.9,<24
     - pandas>=1.0.0,<2
@@ -41,18 +41,19 @@ requirements:
     - s3fs>=2022.11,<2024
     - scikit-learn>=1.2.1,<1.4
     - scipy>=1.9,<2
-    - snowflake-connector-python>=3.0.4,<4
-    - snowflake-snowpark-python>=1.8.0,<2,!=1.12.0
+    - snowflake-connector-python>=3.5.0,<4
+    - snowflake-snowpark-python>=1.11.1,<2,!=1.12.0
     - sqlparse>=0.4,<1
     - typing-extensions>=4.1.0,<5
     - xgboost>=1.7.3,<2
     - python>=3.8,<3.12
   run_constrained:
     - lightgbm==3.3.5
     - mlflow>=2.1.0,<2.4
+    - sentence-transformers>=2.2.2,<3
     - sentencepiece>=0.1.95,<0.2
     - shap==0.42.1
-    - tensorflow>=2.9,<3,!=2.12.0
+    - tensorflow>=2.10,<3
     - tokenizers>=0.10,<1
     - torchdata>=0.4,<1
     - transformers>=4.32.1,<5