tox.ini

[coverage:report]
skip_covered = False
show_missing = True
# TODO: SNOW-1453027 remove omit for config
# TODO: SNOW-1826001 remove mock folder, and bring up coverage.
omit = */src/snowflake/snowpark/modin/config/*
       */src/snowflake/snowpark/modin/plugin/docstrings/*
       */src/snowflake/snowpark/mock/*
[coverage:run]
branch = true
parallel = true
concurrency = multiprocessing, thread
# TODO: SNOW-1453027 remove omit for config
omit = */src/snowflake/snowpark/modin/config/*
       */src/snowflake/snowpark/modin/plugin/docstrings/*
[coverage:paths]
source = src/snowflake/snowpark
         */.tox/*/lib/python*/site-packages/snowflake/snowpark
         */.tox\*\Lib\site-packages\snowflake\snowpark
         */src/snowflake/snowpark
         *\src\snowflake\snowpark
         */fips_env/lib/python*/site-packages/snowflake/snowpark
[coverage:html]
show_contexts = true

[tox]
minversion = 3.7
envlist = fix_lint,
          py39,
          coverage
          nopandas
skip_missing_interpreters = true
setenv = SNOWPARK_LOCAL_TESTING_INTERNAL_TELEMETRY=1

[testenv]
allowlist_externals = bash
description = run the tests with pytest under {basepython}
deps =
    pip >= 19.3.1
    pytest-xdist
    pytest-timeout
    pytest-rerunfailures
    .[pandas]
    .[development]
    .[opentelemetry]
    {env:SNOWFLAKE_PYTEST_MODIN_DEPS}
    {env:SNOWFLAKE_PYTEST_PANDAS_DEPS}
install_command = bash ./scripts/tox_install_cmd.sh {opts} {packages}
setenv =
    COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}}
    ci: SNOWFLAKE_PYTEST_VERBOSITY = -vvv
    # Do not run doctests in parallel so coverage works
    # Snowpark uses 36 workers to accelerate testing in merge gate
    !doctest: SNOWFLAKE_PYTEST_PARALLELISM = -n 36
    # Snowpark uses 4 workers for daily testing since some of its test jobs use weak MacOS instances.
    !doctest: SNOWFLAKE_PYTEST_DAILY_PARALLELISM = -n 6
    # Set test type, either notset, unit, integ, or both
    unit-integ-doctest: SNOWFLAKE_TEST_TYPE = (unit or integ or doctest)
    !unit-!integ-!doctest: SNOWFLAKE_TEST_TYPE = (unit or integ or doctest)
    unit: SNOWFLAKE_TEST_TYPE = unit
    integ: SNOWFLAKE_TEST_TYPE = integ
    doctest: SNOWFLAKE_TEST_TYPE = doctest
    ast: SNOWFLAKE_TEST_TYPE = ast
    # Add common parts into pytest command
    SNOWFLAKE_PYTEST_COV_LOCATION = {env:JUNIT_REPORT_DIR:{toxworkdir}}/junit.{envname}-{env:cloud_provider:dev}.xml
    SNOWFLAKE_PYTEST_COV_CMD = --cov snowflake.snowpark --junitxml {env:SNOWFLAKE_PYTEST_COV_LOCATION} --cov-report=
    SNOWFLAKE_PYTEST_IGNORE_MODIN_CMD = --ignore=src/snowflake/snowpark/modin --ignore=tests/integ/modin --ignore=tests/unit/modin
    SNOWFLAKE_PYTEST_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources {env:SNOWFLAKE_PYTEST_IGNORE_MODIN_CMD}
    SNOWFLAKE_PYTEST_DAILY_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_DAILY_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources {env:SNOWFLAKE_PYTEST_IGNORE_MODIN_CMD}
    # This configures the extra dependency required by modin test
    modin: SNOWFLAKE_PYTEST_MODIN_DEPS = [modin-development]
    modin_pandas_version: SNOWFLAKE_PYTEST_PANDAS_DEPS = pandas=={env:MODIN_PANDAS_PATCH_VERSION}
    SNOW_1314507_WORKAROUND_RERUN_FLAGS = --reruns 5 --reruns-delay 5 --only-rerun "Insufficient resource during interleaved execution."
    MODIN_PYTEST_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources
    MODIN_PYTEST_DAILY_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_DAILY_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources
    MODIN_PYTEST_NO_COV_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} --ignore=tests/resources

passenv =
    AWS_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY
    SF_PROJECT_ROOT
    cloud_provider
    SF_REGRESS_LOGS
    ; Github Actions provided environmental variables
    GITHUB_ACTIONS
    JENKINS_HOME
    ; This is required on windows. Otherwise pwd module won't be imported successfully,
    ; see https://github.com/tox-dev/tox/issues/1455
    USERNAME
    CLIENT_LOG_DIR_PATH_DOCKER
    PYTEST_ADDOPTS
    SNOWFLAKE_IS_PYTHON_RUNTIME_TEST
    snowflake_path
    ; Below only used in AST tests
    TZ
    GITHUB_ENV
commands =
    notudf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} src/snowflake/snowpark tests
    udf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} src/snowflake/snowpark tests
    notdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} tests
    notudfdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} tests
    local: {env:SNOWFLAKE_PYTEST_CMD} --local_testing_mode -m "integ or unit or mock" {posargs:} tests
    dailynotdoctest: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} tests
    dailynotdoctestnotudf: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} tests
    # Snowpark pandas commands:
    snowparkpandasnotdoctest: {env:MODIN_PYTEST_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin tests/integ/test_df_to_snowpark_pandas.py
    # This one only run doctest but we still need to include the tests folder to let tests/conftest.py to mark the doctest files for us
    snowparkpandasdoctest: {env:MODIN_PYTEST_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} src/snowflake/snowpark/modin/ tests/unit/modin
    # This one is used by daily_modin_precommit.yml
    snowparkpandasdailynotdoctest: {env:MODIN_PYTEST_DAILY_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin tests/integ/test_df_to_snowpark_pandas.py
    # This one is only called by jenkins job and the only difference from `snowparkpandasnotdoctest` is that it uses
    # MODIN_PYTEST_NO_COV_CMD instead of MODIN_PYTEST_CMD
    snowparkpandasjenkins: {env:MODIN_PYTEST_NO_COV_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin
    # Snowpark IR commands:
    ast: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} tests

[testenv:nopandas]
allowlist_externals = bash
description = run the tests with pytest under {basepython} with no pandas installed
deps =
    pip >= 19.3.1
    pytest-xdist
    pytest-timeout
    .[development]
    .[opentelemetry]
commands = {env:SNOWFLAKE_PYTEST_DAILY_CMD} -vvv -m "integ or unit" {posargs:}  tests

[testenv:modin_extra_without_pandas_extra]
allowlist_externals = bash
description = SNOW-1345421: test that we can use modin by specifying the modin extra but not the pandas extra.
deps = .[modin-development]
commands = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COV_CMD} {posargs:}  tests/integ/modin/test_modin_extra_without_pandas_extra.py

[testenv:coverage]
description = [run locally after tests]: combine coverage data and create report
deps = {[testenv]deps}
       coverage!=7.6.5
skip_install = True
passenv =
    DIFF_AGAINST
    SNOWFLAKE_IS_PYTHON_RUNTIME_TEST
setenv = COVERAGE_FILE={toxworkdir}/.coverage
commands = coverage combine
           coverage report -m
           coverage xml -o {env:COV_REPORT_DIR:{toxworkdir}}/coverage.xml
           coverage html -d {env:COV_REPORT_DIR:{toxworkdir}}/htmlcov --show-contexts
depends = py39, py310, py311, py312

[testenv:docs]
basepython = python3.9
description = build docs for the project
skip_install = false
deps =
    sphinx
    .[modin-development]
extras = pandas
commands =
    sphinx-build -d "{envtmpdir}{/}doctree" docs/source "{toxworkdir}{/}docs_out" --color -b html {posargs:-W --keep-going}
    python -c 'print(r"documentation available under file://{toxworkdir}{/}docs_out{/}index.html")'

[testenv:flake8]
description = check code style with flake8
skip_install = true
deps = flake8
commands = flake8 {posargs}

[testenv:fix_lint]
allowlist_externals = bash
description = format the code base to adhere to our styles, and complain about what we cannot do automatically
basepython = python3.9
passenv =
    PROGRAMDATA
deps =
    pre-commit >= 3.5.0
skip_install = True
commands = pre-commit run -a -v
           python -c 'import pathlib; print("hint: run \{\} install to add checks as pre-commit hook".format(pathlib.Path(r"{envdir}") / "bin" / "pre-commit"))'

[testenv:pyright]
description = static type checking with pyright
deps = pyright==1.1.338
commands =
    pyright src/snowflake/snowpark/_internal/analyzer
    pyright src/snowflake/snowpark/_internal/compiler
    pyright src/snowflake/snowpark/stored_procedure_profiler.py

[testenv:protoc]
description = generate python code from protobuf
allowlist_externals = bash, protoc
deps =
    protoc-wheel-0==21.1
    mypy-protobuf
    protobuf
commands =
    protoc --proto_path=src/snowflake/snowpark/_internal/proto/ --python_out=src/snowflake/snowpark/_internal/proto/generated --mypy_out=src/snowflake/snowpark/_internal/proto/generated/ src/snowflake/snowpark/_internal/proto/ast.proto

[testenv:dev]
description = create dev environment
extras = pandas, development, sso
usedevelop = True
commands = python -m pip list --format=columns
           python -c "print(r'{envpython}')"

[testenv:snowpark_pandas_py38_import_error]
description = test RuntimeError when importing Snowpark pandas on Python 3.8
basepython = python3.8
commands = python -m pytest --noconftest tests/unit/modin/test_python_version.py

[testenv:snowpark_pandas_old_np]
description = test that Snowpark pandas works with numpy 1.26
deps = .[modin-development]
       numpy<2.0.0
commands = python -m pytest tests/integ/modin/test_old_numpy_aliases.py

[pytest]
log_cli = True
log_cli_level = DEBUG
markers =
    # Optional dependency groups markers
    udf: Snowpark UDF tests
    # Test type markers
    integ: integration tests
    unit: unit tests
    doctest: doctest tests
    # Other markers
    timeout: tests that need a timeout time
    modin_sp_precommit: modin precommit tests run in sproc
    ast: tests run from the ast directory
    compiler: tests run from the compiler directory
    scala: tests run from the scala directory
    mock: tests run from the mock directory
addopts = --doctest-modules --timeout=1200

[flake8]
# E203: Whitespace before ':'
# E501: Line too long
# W503: Line break occurred before a binary operator
ignore = E203,E501,W503
exclude=
    build,.tox,parameters.py,
# Disable checking virtualenv contents
    *venv*
max-line-length = 88
show-source = true