From f06c8fd37a2760bedeed8b930e8c1d9336ae956c Mon Sep 17 00:00:00 2001 From: Xingzhi Zhang <37076709+elliotzh@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:58:09 +0800 Subject: [PATCH] feat: enable ci for the new example --- .../samples_flowinpipeline_pipeline.yml | 47 ++++++++++++++ .../azureai/use-flow-in-azure-ml-pipeline.md} | 64 +++++++++++++++++-- docs/cloud/index.md | 2 +- docs/how-to-guides/init-and-test-a-flow.md | 1 - examples/README.md | 2 + examples/configuration.ipynb | 34 ++++++++++ .../tutorials/flow-in-pipeline/pipeline.ipynb | 12 ++-- 7 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/samples_flowinpipeline_pipeline.yml rename docs/{how-to-guides/use-flow-in-pipeline.md => cloud/azureai/use-flow-in-azure-ml-pipeline.md} (69%) diff --git a/.github/workflows/samples_flowinpipeline_pipeline.yml b/.github/workflows/samples_flowinpipeline_pipeline.yml new file mode 100644 index 000000000000..4699878019be --- /dev/null +++ b/.github/workflows/samples_flowinpipeline_pipeline.yml @@ -0,0 +1,47 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: samples_flowinpipeline_pipeline +on: + schedule: + - cron: "28 19 * * *" # Every day starting at 3:28 BJT + pull_request: + branches: [ main ] + paths: [ examples/**, .github/workflows/samples_flowinpipeline_pipeline.yml ] + workflow_dispatch: + +jobs: + samples_notebook_ci: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}" + - name: Test Notebook + working-directory: examples/tutorials/flow-in-pipeline + run: | + papermill -k python pipeline.ipynb pipeline.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples/tutorials/flow-in-pipeline diff --git a/docs/how-to-guides/use-flow-in-pipeline.md b/docs/cloud/azureai/use-flow-in-azure-ml-pipeline.md similarity index 69% rename from docs/how-to-guides/use-flow-in-pipeline.md rename to docs/cloud/azureai/use-flow-in-azure-ml-pipeline.md index f1973b09d654..d4871b7110b7 100644 --- a/docs/how-to-guides/use-flow-in-pipeline.md +++ b/docs/cloud/azureai/use-flow-in-azure-ml-pipeline.md @@ -1,10 +1,6 @@ -# Use flow in pipeline job +# Use flow in Azure ML pipeline job -:::{admonition} Experimental feature -This is an experimental feature, and may change at any time. Learn [more](faq.md#stable-vs-experimental). -::: - -After you have developed and tested the flow in [init and test a flow](./init-and-test-a-flow.md), this guide will help you learn how to use a flow as a parallel component in a pipeline job on AzureML, so that you can integrate the created flow with existing pipelines and process a large amount of data. +After you have developed and tested the flow in [init and test a flow](../../how-to-guides/init-and-test-a-flow.md), this guide will help you learn how to use a flow as a parallel component in a pipeline job on AzureML, so that you can integrate the created flow with existing pipelines and process a large amount of data. :::{admonition} Pre-requirements - Customer need to install the extension `ml>=2.21.0` to enable this feature in CLI and package `azure-ai-ml>=1.11.0` to enable this feature in SDK; @@ -76,4 +72,58 @@ Besides explicitly registering a flow as a component, customer can also directly - [SDK example](https://github.com/Azure/azureml-examples/tree/main/sdk/python/jobs/pipelines/1l_flow_in_pipeline) All connections and flow inputs will be exposed as input parameters of the component. Default value can be provided in flow/run definition; they can also be set/overwrite on job submission: -- [CLI/SDK example](../../examples/tutorials/flow-in-pipeline/pipeline.ipynb) + +::::{tab-set} +:::{tab-item} CLI +:sync: CLI + +```yaml +... +jobs: + flow_node: + type: parallel + component: standard/web-classification/flow.dag.yaml + inputs: + data: ${{parent.inputs.web_classification_input}} + url: "${data.url}" + connections.summarize_text_content.connection: azure_open_ai_connection + connections.summarize_text_content.deployment_name: text-davinci-003 +... +``` + +::: + +:::{tab-item} SDK +:sync: SDK + +```python +from azure.ai.ml import dsl + +ml_client = MLClient() + +# Register flow as a component +flow_component = load_component("standard/web-classification/flow.dag.yaml") +data_input = Input(path="standard/web-classification/data.jsonl", type=AssetTypes.URI_FILE) + +@dsl.pipeline +def pipeline_func_with_flow(data): + flow_node = flow_component( + data=data, + url="${data.url}", + connections={ + "summarize_text_content": { + "connection": "azure_open_ai_connection", + "deployment_name": "text-davinci-003", + }, + }, + ) + flow_node.compute = "cpu-cluster" + +pipeline_with_flow = pipeline_func_with_flow(data=data_input) + +ml_client.jobs.create_or_update(pipeline_with_flow) +``` + +::: + +:::: diff --git a/docs/cloud/index.md b/docs/cloud/index.md index da3340301bd6..2c8aa2e2584b 100644 --- a/docs/cloud/index.md +++ b/docs/cloud/index.md @@ -7,6 +7,6 @@ You can develope your flow locally and seamlessly move the experience to azure c :maxdepth: 1 azureai/quick-start azureai/deploy-to-azure-appservice -azureai/flow-in-pipeline +azureai/use-flow-in-azure-ml-pipeline.md azureai/faq ``` \ No newline at end of file diff --git a/docs/how-to-guides/init-and-test-a-flow.md b/docs/how-to-guides/init-and-test-a-flow.md index 0ac0be59bf1d..9e157a83f185 100644 --- a/docs/how-to-guides/init-and-test-a-flow.md +++ b/docs/how-to-guides/init-and-test-a-flow.md @@ -300,4 +300,3 @@ Break points and debugging functionalities for the Python steps in your flow. Ju - [Add conditional control to a flow](./add-conditional-control-to-a-flow.md) - [Run and evaluate a flow](./run-and-evaluate-a-flow.md) -- [Use flow in pipeline job](./use-flow-in-pipeline.md) diff --git a/examples/README.md b/examples/README.md index 107c30947250..41617e99a4e6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -28,6 +28,7 @@ ------|--------|------------- | [chat-with-pdf](tutorials/e2e-development/chat-with-pdf.md) | [![samples_tutorials_e2e_development_chat_with_pdf](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_e2e_development_chat_with_pdf.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_e2e_development_chat_with_pdf.yml) | Retrieval Augmented Generation (or RAG) has become a prevalent pattern to build intelligent application with Large Language Models (or LLMs) since it can infuse external knowledge into the model, which is not trained with those up-to-date or proprietary information | | [azure-app-service](tutorials/flow-deploy/azure-app-service/README.md) | [![samples_tutorials_flow_deploy_azure_app_service](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_azure_app_service.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_azure_app_service.yml) | This example demos how to deploy a flow using Azure App Service | +| [distribute-flow-as-executable-app](tutorials/flow-deploy/distribute-flow-as-executable-app/README.md) | [![samples_tutorials_flow_deploy_distribute_flow_as_executable_app](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_distribute_flow_as_executable_app.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_distribute_flow_as_executable_app.yml) | This example demos how to package flow as a executable app | | [docker](tutorials/flow-deploy/docker/README.md) | [![samples_tutorials_flow_deploy_docker](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_docker.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_docker.yml) | This example demos how to deploy flow as a docker app | | [kubernetes](tutorials/flow-deploy/kubernetes/README.md) | [![samples_tutorials_flow_deploy_kubernetes](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_kubernetes.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_tutorials_flow_deploy_kubernetes.yml) | This example demos how to deploy flow as a Kubernetes app | @@ -87,6 +88,7 @@ ------|--------|------------- | [quickstart.ipynb](tutorials/get-started/quickstart.ipynb) | [![samples_getstarted_quickstart](https://github.com/microsoft/promptflow/actions/workflows/samples_getstarted_quickstart.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_getstarted_quickstart.yml) | A quickstart tutorial to run a flow and evaluate it. | | [quickstart-azure.ipynb](tutorials/get-started/quickstart-azure.ipynb) | [![samples_getstarted_quickstartazure](https://github.com/microsoft/promptflow/actions/workflows/samples_getstarted_quickstartazure.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_getstarted_quickstartazure.yml) | A quickstart tutorial to run a flow in Azure AI and evaluate it. | +| [pipeline.ipynb](tutorials/flow-in-pipeline/pipeline.ipynb) | [![samples_flowinpipeline_pipeline](https://github.com/microsoft/promptflow/actions/workflows/samples_flowinpipeline_pipeline.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_flowinpipeline_pipeline.yml) | {'description': 'Create pipeline using components to run a distributed job with tensorflow'} | | [cloud-run-management.ipynb](tutorials/run-management/cloud-run-management.ipynb) | [![samples_runmanagement_cloudrunmanagement](https://github.com/microsoft/promptflow/actions/workflows/samples_runmanagement_cloudrunmanagement.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_runmanagement_cloudrunmanagement.yml) | Flow run management in Azure AI | | [connection.ipynb](connections/connection.ipynb) | [![samples_connections_connection](https://github.com/microsoft/promptflow/actions/workflows/samples_connections_connection.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_connections_connection.yml) | Manage various types of connections using sdk | | [chat-with-pdf-azure.ipynb](flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb) | [![samples_flows_chat_chatwithpdf_chatwithpdfazure](https://github.com/microsoft/promptflow/actions/workflows/samples_flows_chat_chatwithpdf_chatwithpdfazure.yml/badge.svg?branch=main)](https://github.com/microsoft/promptflow/actions/workflows/samples_flows_chat_chatwithpdf_chatwithpdfazure.yml) | A tutorial of chat-with-pdf flow that executes in Azure AI | diff --git a/examples/configuration.ipynb b/examples/configuration.ipynb index c993738e4aaf..158f571b06db 100644 --- a/examples/configuration.ipynb +++ b/examples/configuration.ipynb @@ -121,6 +121,40 @@ "print(pf)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Retrieve or create an Azure Machine Learning compute target\n", + "\n", + "To create a Azure Machine Learning job, you need a compute cluster as prerequisite. Below code ensures computes named `cpu-cluster` exists in your workspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "from azure.ai.ml.entities import AmlCompute\n", + "\n", + "# MLClient use the same configuration as PFClient\n", + "ml_client = MLClient.from_config(credential=credential)\n", + "\n", + "# specify aml compute name.\n", + "cpu_compute_target = \"cpu-cluster\"\n", + "\n", + "try:\n", + " ml_client.compute.get(cpu_compute_target)\n", + "except Exception:\n", + " print(\"Creating a new cpu compute target...\")\n", + " compute = AmlCompute(\n", + " name=cpu_compute_target, size=\"STANDARD_D2_V2\", min_instances=0, max_instances=4\n", + " )\n", + " ml_client.compute.begin_create_or_update(compute).result()" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/tutorials/flow-in-pipeline/pipeline.ipynb b/examples/tutorials/flow-in-pipeline/pipeline.ipynb index bc7489998e9f..3208e8659491 100644 --- a/examples/tutorials/flow-in-pipeline/pipeline.ipynb +++ b/examples/tutorials/flow-in-pipeline/pipeline.ipynb @@ -12,11 +12,10 @@ "- An Azure ML workspace with computer cluster - [Configure workspace](../../configuration.ipynb)\n", "- A python environment\n", "- Installed Azure Machine Learning Python SDK v2 - [install instructions](../../../README.md) - check the getting started section\n", - "- **Permission to access connections has been assigned to the used compute cluster**\n", "\n", "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", "- Connect to your AML workspace from the Python SDK\n", - "- Create `Pipeline` load flow as components from YAML\n", + "- Create `Pipeline` with a component loaded from `flow.dag.yaml`\n", "\n", "**Motivations** - This notebook explains how to run a pipeline with distributed training component." ] @@ -43,8 +42,7 @@ "\n", "from azure.ai.ml import MLClient, load_component, Input\n", "from azure.ai.ml.constants import AssetTypes\n", - "from azure.ai.ml.dsl import pipeline\n", - "from azure.ai.ml.entities import ResourceConfiguration" + "from azure.ai.ml.dsl import pipeline" ] }, { @@ -93,7 +91,7 @@ "ml_client = MLClient.from_config(credential=credential)\n", "\n", "# Retrieve an already attached Azure Machine Learning Compute.\n", - "cluster_name = \"gpu-cluster\"\n", + "cluster_name = \"cpu-cluster\"\n", "print(ml_client.compute.get(cluster_name))" ] }, @@ -112,7 +110,7 @@ "metadata": {}, "outputs": [], "source": [ - "flow_component = load_component(\"./web_classification/flow.dag.yaml\")" + "flow_component = load_component(\"../standard/web-classification/flow.dag.yaml\")" ] }, { @@ -129,7 +127,7 @@ "metadata": {}, "outputs": [], "source": [ - "data_input = Input(path=\"./data/data.jsonl\", type=AssetTypes.URI_FILE)\n", + "data_input = Input(path=\"../standard/web-classification/data.jsonl\", type=AssetTypes.URI_FILE)\n", "\n", "@pipeline()\n", "def pipeline_func_with_flow(data):\n",