Add external access templates for snowpark and streamlit (#13)

* Add external access templates for snowpark and streamlit

* Add automatic check for V2 vs CLI version limit

* style fixes

* fix templates

* limit checked .yml files to snowflake.ymls

* Remove pycache

---------

Co-authored-by: Patryk Czajka <[email protected]>

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+.idea
+*.pyc

.tests/test_V2_definition_has_cli_version_limit.py

@@ -0,0 +1,37 @@
+import re
+from pathlib import Path
+
+import pytest
+
+_REPO_ROOT = Path(__file__).parent.parent
+
+
+def iter_all_templates():
+    return (
+        # "str" to make tests parameters human-readable
+        str(x.relative_to(_REPO_ROOT).parent)
+        for x in _REPO_ROOT.rglob("**/template.yml")
+    )
+
+
+def template_has_cli_version_limit(template_root: Path) -> bool:
+    return "minimum_cli_version" in (template_root / "template.yml").read_text()
+
+
+def is_snowflake_yml_V2(template_root: Path) -> bool:
+    for file in template_root.rglob("snowflake.yml"):
+        for line in file.read_text().splitlines():
+            if re.match(r".*definition_version:\s+.?2.*", line):
+                return True
+    return False
+
+
+@pytest.mark.parametrize("template_root", iter_all_templates())
+def test_V2_template_has_cli_version_limit(template_root):
+    template_path = _REPO_ROOT / template_root
+    if not is_snowflake_yml_V2(template_path):
+        pytest.skip("No snowflake.yml in definition version 2 found")
+
+    assert template_has_cli_version_limit(
+        template_path
+    ), "snowflake.yml V2 is not supported in Snowflake CLI 2.X. Please add 'minimum_cli_version: 3.0.0' to template.yml"

.tests/test_examples.py

@@ -1,10 +1,11 @@
 import os
 import subprocess
+from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from snowflake.cli.__about__ import VERSION
+
 import pytest
-from contextlib import contextmanager
+from snowflake.cli.__about__ import VERSION
 
 if VERSION < "2.8.0":
     pytest.skip("This test requires CLI >= 2.8.0", allow_module_level=True)

.tests/test_render_templates.py

@@ -5,8 +5,8 @@
 
 import pytest
 import yaml
-from snowflake.cli.api.project.schemas.template import Template
 from snowflake.cli.__about__ import VERSION
+from snowflake.cli.api.project.schemas.template import Template
 
 _REPO_ROOT = Path(__file__).parent.parent
 from packaging.version import parse

snowpark_with_external_access/.gitignore

@@ -0,0 +1,4 @@
+.packages/
+.venv/
+app.zip
+__pycache__

snowpark_with_external_access/README.md

@@ -0,0 +1,49 @@
+# Snowpark project using external access
+
+This is a simple example of a Snowpark project that requires external access.
+
+## Prerequisites
+This project requires a database, API integration and secret. To created them you can execute convenience script
+`snow sql -f setup.sql` or run the following SQL commands:
+
+```sql
+CREATE DATABASE IF NOT EXISTS <! database_name | to_snowflake_identifier !>;
+CREATE SCHEMA IF NOT EXISTS <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+USE SCHEMA <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+CREATE SECRET IF NOT EXISTS <! secret_name | to_snowflake_identifier !> TYPE = GENERIC_STRING SECRET_STRING = 'very_secret_string';
+CREATE OR REPLACE NETWORK RULE snowpark_example_network_rule
+  MODE = EGRESS
+  TYPE = HOST_PORT
+  VALUE_LIST = ('docs.snowflake.com');
+
+CREATE EXTERNAL ACCESS INTEGRATION IF NOT EXISTS <! external_access_integration_name !>
+  ALLOWED_NETWORK_RULES = (snowpark_example_network_rule)
+  ALLOWED_AUTHENTICATION_SECRETS = (<! secret_name | to_snowflake_identifier !>)
+  ENABLED = true;
+```
+
+## Building Snowpark artifacts
+_For more information see [build documentation](https://docs.snowflake.com/developer-guide/snowflake-cli/snowpark/build)._
+
+First you need to bundle your code by running:
+```bash
+snow snowpark build
+```
+
+## Deploying the project
+_For more information see [deploy documentation](https://docs.snowflake.com/developer-guide/snowflake-cli/snowpark/deploy)._
+
+To deploy the snowpark application:
+
+```bash
+snow snowpark deploy
+```
+
+## Testing the project
+
+You can test the deployed snowpark application by running:
+
+```bash
+snow snowpark execute function "<! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>.request_function()";
+snow snowpark execute procedure "<! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>.request_procedure()";
+```

snowpark_with_external_access/app/common.py

@@ -0,0 +1,14 @@
+import _snowflake
+from http.client import HTTPSConnection
+
+
+def get_secret_value():
+    return _snowflake.get_generic_secret_string("generic_secret")
+
+
+def send_request():
+    host = "docs.snowflake.com"
+    conn = HTTPSConnection(host)
+    conn.request("GET", "/")
+    response = conn.getresponse()
+    return response.status

snowpark_with_external_access/app/functions.py

@@ -0,0 +1,9 @@
+from common import get_secret_value, send_request
+
+
+def request_function() -> str:
+    # Retrieve secret value
+    _ = get_secret_value()
+
+    # Send request
+    return send_request()

snowpark_with_external_access/app/procedures.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from snowflake.snowpark import Session
+from common import get_secret_value, send_request
+
+
+def request_procedure(session: Session) -> str:
+    # Retrieve secret value
+    _ = get_secret_value()
+
+    # Send request
+    return send_request()

snowpark_with_external_access/requirements.txt

@@ -0,0 +1 @@
+snowflake-snowpark-python

snowpark_with_external_access/setup.sql

@@ -0,0 +1,15 @@
+CREATE DATABASE IF NOT EXISTS <! database_name | to_snowflake_identifier !>;
+CREATE SCHEMA IF NOT EXISTS <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+USE SCHEMA <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+
+CREATE SECRET IF NOT EXISTS <! secret_name | to_snowflake_identifier !> TYPE = GENERIC_STRING SECRET_STRING = 'very_secret_string';
+
+CREATE OR REPLACE NETWORK RULE snowpark_example_network_rule
+  MODE = EGRESS
+  TYPE = HOST_PORT
+  VALUE_LIST = ('docs.snowflake.com');
+
+CREATE EXTERNAL ACCESS INTEGRATION IF NOT EXISTS <! external_access_integration_name !>
+  ALLOWED_NETWORK_RULES = (snowpark_example_network_rule)
+  ALLOWED_AUTHENTICATION_SECRETS = (<! secret_name | to_snowflake_identifier !>)
+  ENABLED = true;

snowpark_with_external_access/snowflake.yml

@@ -0,0 +1,52 @@
+# For more information about structure of snowflake.yml for Snowpark see
+# https://docs.snowflake.com/en/developer-guide/snowflake-cli/snowpark/create
+definition_version: '2'
+entities:
+  request_function:
+    type: function
+    # Uses context variables to create fully qualified name of the function
+    identifier:
+      name: request_function
+      schema: <% ctx.env.schema %>
+      database: <% ctx.env.database %>
+    handler: functions.request_function
+    returns: string
+    # No arguments for this function
+    signature: ""
+    meta:
+      use_mixins:
+        - external_access
+        - snowpark_shared
+
+  request_procedure:
+    type: procedure
+    # Uses context variables to create fully qualified name of the procedure
+    identifier:
+      name: request_procedure
+      schema: <% ctx.env.schema %>
+      database: <% ctx.env.database %>
+    handler: procedures.request_procedure
+    returns: string
+    # No arguments for this procedure
+    signature: ""
+    meta:
+      use_mixins:
+        - external_access
+        - snowpark_shared
+
+mixins:
+  # This mixin defines shared configuration for external access
+  external_access:
+    secrets:
+      # generic_secret is key used by the get_secret_value method to reference the secret
+      generic_secret: <! secret_name | to_snowflake_identifier !>
+    external_access_integrations:
+      - <! external_access_integration_name !>
+  snowpark_shared:
+    artifacts:
+      - app/
+    stage: <% ctx.env.database %>.<% ctx.env.schema %>.<! stage_name | to_snowflake_identifier !>
+
+env:
+  schema: <! schema_name | to_snowflake_identifier !>
+  database: <! database_name | to_snowflake_identifier !>

snowpark_with_external_access/template.yml

@@ -0,0 +1,25 @@
+minimum_cli_version: "3.0.0"
+files_to_render:
+  - snowflake.yml
+  - README.md
+  - setup.sql
+variables:
+  - name: database_name
+    prompt: "Database where the functions and procedures will be created"
+    type: string
+  - name: schema_name
+    prompt: "Schema where the functions and procedures will be created"
+    type: string
+    default: public
+  - name: stage_name
+    prompt: "What stage should the functions and procedures be deployed to"
+    default: dev_deployment
+    type: string
+  - name: secret_name
+    prompt: "Secret name to be used in the functions and procedures"
+    type: string
+    default: snowpark_secret
+  - name: external_access_integration_name
+    prompt: "External access integration name to be used in the functions and procedures"
+    type: string
+    default: snowpark_external_access_integration

streamlit_with_external_access/.gitignore

@@ -0,0 +1,4 @@
+.packages/
+.venv/
+app.zip
+__pycache__

streamlit_with_external_access/README.md

@@ -0,0 +1,34 @@
+# Streamlit application with external access
+
+This is a simple example of a Streamlit application that requires external access.
+
+## Prerequisites
+This project requires a database, API integration and secret. To created them you can execute convenience script
+`snow sql -f setup.sql` or run the following SQL commands:
+
+```sql
+CREATE DATABASE IF NOT EXISTS <! database_name | to_snowflake_identifier !>;
+CREATE SCHEMA IF NOT EXISTS <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+USE SCHEMA <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+
+CREATE SECRET IF NOT EXISTS <! secret_name | to_snowflake_identifier !> TYPE = GENERIC_STRING SECRET_STRING = 'very_secret_string';
+
+CREATE OR REPLACE NETWORK RULE streamlit_example_network_rule
+  MODE = EGRESS
+  TYPE = HOST_PORT
+  VALUE_LIST = ('docs.snowflake.com');
+
+CREATE EXTERNAL ACCESS INTEGRATION IF NOT EXISTS <! external_access_integration_name !>
+  ALLOWED_NETWORK_RULES = (streamlit_example_network_rule)
+  ALLOWED_AUTHENTICATION_SECRETS = (<! secret_name | to_snowflake_identifier !>)
+  ENABLED = true;
+```
+
+## Deploying the streamlit application
+_For more information see [deploy documentation](https://docs.snowflake.com/developer-guide/snowflake-cli/streamlit-apps/manage-apps/deploy-app)._
+
+To deploy the Streamlit application, you should run:
+
+```bash
+snow streamlit deploy
+```

streamlit_with_external_access/requirements.txt

@@ -0,0 +1 @@
+snowflake-snowpark-python

streamlit_with_external_access/setup.sql

@@ -0,0 +1,15 @@
+CREATE DATABASE IF NOT EXISTS <! database_name | to_snowflake_identifier !>;
+CREATE SCHEMA IF NOT EXISTS <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+USE SCHEMA <! database_name | to_snowflake_identifier !>.<! schema_name | to_snowflake_identifier !>;
+
+CREATE SECRET IF NOT EXISTS <! secret_name | to_snowflake_identifier !> TYPE = GENERIC_STRING SECRET_STRING = 'very_secret_string';
+
+CREATE OR REPLACE NETWORK RULE streamlit_example_network_rule
+  MODE = EGRESS
+  TYPE = HOST_PORT
+  VALUE_LIST = ('docs.snowflake.com');
+
+CREATE EXTERNAL ACCESS INTEGRATION IF NOT EXISTS <! external_access_integration_name !>
+  ALLOWED_NETWORK_RULES = (streamlit_example_network_rule)
+  ALLOWED_AUTHENTICATION_SECRETS = (<! secret_name | to_snowflake_identifier !>)
+  ENABLED = true;

streamlit_with_external_access/snowflake.yml

@@ -0,0 +1,35 @@
+# For more information about structure of snowflake.yml for Streamlit see
+# https://docs.snowflake.com/developer-guide/snowflake-cli/streamlit-apps/manage-apps/initialize-app#create-the-project-definition-for-a-streamlit-app
+definition_version: '2'
+entities:
+  dashboard:
+    type: 'streamlit'
+    # Uses context variables to create fully qualified name of the dashboard
+    identifier:
+      name: 'dashboard'
+      schema: <% ctx.env.schema %>
+      database: <% ctx.env.database %>
+    query_warehouse: <! warehouse_name !>
+    artifacts:
+      - streamlit_app.py
+    meta:
+      use_mixins:
+        - external_access
+        - deployment_stage
+
+mixins:
+  # This mixin defines shared configuration for external access
+  external_access:
+    secrets:
+      # generic_secret is key used by get_secret_value method to reference the secret
+      generic_secret: <! secret_name | to_snowflake_identifier !>
+    external_access_integrations:
+      - <! external_access_integration_name !>
+
+  deployment_stage:
+    # Uses context variables to create fully qualified name of stage
+    stage: <% ctx.env.database %>.<% ctx.env.schema %>.<! stage_name | to_snowflake_identifier !>
+
+env:
+  schema: <! schema_name | to_snowflake_identifier !>
+  database: <! database_name | to_snowflake_identifier !>

streamlit_with_external_access/streamlit_app.py

@@ -0,0 +1,19 @@
+import _snowflake
+import streamlit as st
+from http.client import HTTPSConnection
+
+
+def get_secret_value():
+    return _snowflake.get_generic_secret_string("generic_secret")
+
+
+def send_request():
+    host = "docs.snowflake.com"
+    conn = HTTPSConnection(host)
+    conn.request("GET", "/")
+    response = conn.getresponse()
+    st.success(f"Response status: {response.status}")
+
+
+st.title(f"Example streamlit app.")
+st.button("Send request", on_click=send_request)

streamlit_with_external_access/template.yml

@@ -0,0 +1,28 @@
+minimum_cli_version: "3.0.0"
+files_to_render:
+  - snowflake.yml
+  - README.md
+  - setup.sql
+variables:
+  - name: database_name
+    prompt: "Database where the application will be created"
+    type: string
+  - name: schema_name
+    prompt: "Schema where the application will be created"
+    type: string
+    default: public
+  - name: warehouse_name
+    prompt: "Warehouse to be used by the application"
+    type: string
+  - name: stage_name
+    prompt: "What stage should the procedures and functions be deployed to"
+    default: dev_deployment
+    type: string
+  - name: secret_name
+    prompt: "Secret name to be used in the procedures and functions"
+    type: string
+    default: streamlit_secret
+  - name: external_access_integration_name
+    prompt: "External access integration name to be used in the procedures and functions"
+    type: string
+    default: streamlit_external_access_integration