diff --git a/snowflake_tasks/README.md b/snowflake_tasks/README.md index edcc6c6..d41c7d2 100644 --- a/snowflake_tasks/README.md +++ b/snowflake_tasks/README.md @@ -36,7 +36,11 @@ NOTE: This code package does not yet handle Object-specific loading window. You - Run the script SF_DAG_DEPLOY_OBJECTS.sql in your target Snowflake database. - - In VaultSpeed, set up a Data Vault with FMC_TYPE = Generic and ETL Generation type = Snowflake SQL. Make sure you have generated all the FMC flows created. Be sure to use a valid schedule interval provided in the Snowflake Tasks documentation. Do not enclose the schedule interval value in double quotes (") or single quotes ('). + - In VaultSpeed, set up a Data Vault with FMC_TYPE = generic and ETL Generation type = Snowflake SQL. + + - Generate the DDL and ETL for your Data Vault and deploy both to the target database. + + - Generate the FMC workflows. Be sure to use a valid schedule interval provided in the Snowflake Tasks documentation. Do not enclose the schedule interval value in double quotes (") or single quotes ('). Make sure Group Tasks is toggled off. (VaultSpeed will require a numeric Concurrency value, but it will not impact the workflow. # Usage After completing the prerequisites, go the Automatic Deployment screen in your VaultSpeed subscription. Select the generic FMC generation you wish to deploy and click the up arrow to deploy it. Select the Custom Script option and click deploy. \ No newline at end of file diff --git a/snowflake_tasks/snowsql_deploy.sh b/snowflake_tasks/snowsql_deploy.sh index 9d2d16a..2392986 100644 --- a/snowflake_tasks/snowsql_deploy.sh +++ b/snowflake_tasks/snowsql_deploy.sh @@ -41,20 +41,30 @@ snowsql -c $snowsql_conn -o exit_on_error=true -q "TRUNCATE TASKER.TASK_MAPPING; ## Insert JSON mapping into task mapping table; use as work table for task generation procedure snowsql -c $snowsql_conn -o exit_on_error=true -q "INSERT INTO TASKER.TASK_MAPPING (JSON_MAPPING) SELECT TO_VARIANT(PARSE_JSON('$fmc_json_mapping'));" -## Remove VaultSpeed ASCII header from info JSON +## Remove the "logo" and "comment" from the info JSON fmc_json_text=$(cat $path_to_generated_files/${zipname%%.*}/*FMC_info*.json) dv_find_str='"dv_code"': pos=$(awk -v a="$fmc_json_text" -v b="$dv_find_str" 'BEGIN{print index(a,b)}') fmc_json_text={"${fmc_json_text:$pos+33}" -## Get DAG name and scheduling parameters from info JSON +## Get DAG name, schedule interval, group tasks setting, and target database type from info JSON dag_name=$(echo $fmc_json_text|jq -r '.dag_name') schedule_interval=$(echo $fmc_json_text|jq -r '.schedule_interval') +group_tasks=$(echo $fmc_json_text|jq -r '.group_tasks') +dv_database_type=$(echo $fmc_json_text|jq -r '.dv_database_type') -##Add a default schedule if no value is available in schedule_interval (required for Snowflake Tasks) -if [ "$schedule_interval" == "" ] +##Check for Grouped tasks +if [ "$group_tasks" == "true" ] then - schedule_interval=$default_schedule + echo "Group tasks must be set to 'false' for Snowflake Tasks deployed with generic FMC." >&2 + exit 1 +fi + +##Check target database type +if [ "$dv_database_type" != "SNOWFLAKE" ] +then + echo "The data vault database type must be 'Snowflake' for Snowflake Tasks deployed with generic FMC." >&2 + exit 2 fi ## Execute procedure to generate tasks/dag