diff --git a/plugin/skills/microsoft-foundry/agent/deploy/agent-framework-deploy/SKILL.md b/plugin/skills/microsoft-foundry/agent/deploy/agent-framework-deploy/SKILL.md new file mode 100644 index 000000000..e715262aa --- /dev/null +++ b/plugin/skills/microsoft-foundry/agent/deploy/agent-framework-deploy/SKILL.md @@ -0,0 +1,172 @@ +--- +name: agent-framework-deploy +description: | + Deploy AI agents and workflows built with Microsoft Agent Framework SDK to Microsoft Foundry. Handles Dockerfile generation, ACR container build, hosted agent version creation, and container startup. + USE FOR: deploy agent to Foundry, publish agent to production, host agent on Azure, deploy workflow to Foundry, go live with agent, production deployment agent, deploy agent framework app, push agent to Azure AI Foundry, containerize agent, deploy multi-agent workflow. + DO NOT USE FOR: creating agents (use agent-framework), deploying models (use models/deploy-model), deploying web apps or functions (use azure-deploy), managing quotas (use quota). +--- + +### Deployment + +#### Prerequisites + +- Azure CLI installed and authenticated +- Azure AI Foundry Resource and Project created +- Agent must be wrapped as HTTP server using Agent-as-Server pattern. See [agent-as-server.md](references/agent-as-server.md). + + +#### Deployment Workflow + +**Step 1: Verify locally** + +Verify agent runs locally as HTTP server. + +**Step 2: Gather Context** + +Use az CLI to check the logged-in context first; ask user only for values that cannot be retrieved: +- `SUB_ID` - Azure Subscription ID +- `RG_NAME` - Resource Group name +- `FOUNDRY_RESOURCE` - Azure AI Foundry resource name +- `PROJECT_NAME` - Foundry project name +- `AGENT_NAME` - name for the hosted agent +- `ENTRYPOINT` - main file to run, e.g., `app.py` (if Dockerfile needs to be generated) + + +**Step 3: Prepare Dockerfile** + +If no Dockerfile exists, generate one. Below is a Python example: + +```dockerfile +FROM python:3.11-slim + +WORKDIR /app + +COPY ./ user_agent/ + +WORKDIR /app/user_agent + +RUN if [ -f requirements.txt ]; then \ + pip install -r requirements.txt; \ + else \ + echo "No requirements.txt found"; \ + fi + +EXPOSE 8088 + +CMD ["python", ""] +``` + +Replace `` with user's specified entry point. + +> **Important:** If you decide to add `.env` to `.dockerignore`, confirm with the user which environment variables need to be included in the Docker container. + +**Step 4: Configure ACR** + +**If ACR exists**: Use existing ACR directly, skip role assignment. + +**If no ACR exists**: Create new ACR with ABAC repository permissions mode, then assign: +- **Container Registry Repository Reader** to Foundry project managed identity +- **Container Registry Repository Writer** to current user + +**Step 5: Build and Push Container Image** + +Use ACR remote build (no local Docker required): + +1. Generate a random alphanumeric image tag (12 characters) +2. Build and push the image: +```bash +IMAGE_NAME="$ACR_NAME.azurecr.io/$AGENT_NAME:$IMAGE_TAG" +az acr build --registry $ACR_NAME --image $IMAGE_NAME --subscription $SUB_ID --source-acr-auth-id "[caller]" +``` + +**Important**: `--source-acr-auth-id "[caller]"` is required. + +**Note**: `az acr build` streams logs from the remote build. If the CLI crashes while displaying logs (e.g., `UnicodeEncodeError` on Windows), the remote build continues running independently. Do + not assume failure. Check actual build status with: + ```bash + az acr task show-run -r $ACR_NAME --run-id --query status + ``` + +**Step 6: Deploy Agent Version** + +Ask the user for resource allocation preferences: +- `CPU` - CPU cores for the agent container (default: `0.5`) +- `MEMORY` - Memory allocation (default: `1Gi`) + +Get access token and create agent version: + +```bash +az account get-access-token --resource https://ai.azure.com --query accessToken --output tsv +``` + +``` +POST https://$FOUNDRY_RESOURCE.services.ai.azure.com/api/projects/$PROJECT_NAME/agents/$AGENT_NAME/versions?api-version=2025-05-15-preview +``` + +**Body**: +```json +{ + "definition": { + "kind": "hosted", + "container_protocol_versions": [ + { "protocol": "RESPONSES", "version": "v1" } + ], + "cpu": "$CPU", + "memory": "$MEMORY", + "image": "$IMAGE_NAME", + "environment_variables": { "LOG_LEVEL": "debug" } + } +} +``` + +Record `$AGENT_VERSION` from response. + +**Step 7: Start Agent Container** + +``` +POST https://$FOUNDRY_RESOURCE.services.ai.azure.com/api/projects/$PROJECT_NAME/agents/$AGENT_NAME/versions/$AGENT_VERSION/containers/default:start?api-version=2025-05-15-preview +``` + +**Body**: +```json +{ + "min_replicas": 1, + "max_replicas": 1 +} +``` + +**Step 8: Validate Deployment** + +Test the agent: + +``` +POST https://$FOUNDRY_RESOURCE.services.ai.azure.com/api/projects/$PROJECT_NAME/openai/responses?api-version=2025-05-15-preview +``` + +**Body**: +```json +{ + "agent": { + "type": "agent_reference", + "name": "$AGENT_NAME", + "version": "$AGENT_VERSION" + }, + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hi, what can you do" + } + ] + } + ], + "stream": false +} +``` + +**Step 9: Post-Deployment** + +- Create reusable deployment script in project root +- Save deployment summary to `.azure/foundry-deployment-summary.md` diff --git a/plugin/skills/microsoft-foundry/agent/deploy/agent-framework-deploy/references/agent-as-server.md b/plugin/skills/microsoft-foundry/agent/deploy/agent-framework-deploy/references/agent-as-server.md new file mode 100644 index 000000000..19c60c6f0 --- /dev/null +++ b/plugin/skills/microsoft-foundry/agent/deploy/agent-framework-deploy/references/agent-as-server.md @@ -0,0 +1,83 @@ +# Agent as HTTP Server Best Practices + +Converting an Agent-Framework-based Agent/Workflow/App to run as an HTTP server requires code changes to host the agent as a RESTful HTTP server. + +(This doc applies to Python SDK only) + +## Code Changes + +### Run Workflow as Agent + +Agent Framework provides a way to run a whole workflow as agent, via appending `.as_agent()` to the `WorkflowBuilder`, like: + +```python +agent = ( + WorkflowBuilder() + .add_edge(...) + ... + .set_start_executor(...) + .build() + .as_agent() # here it is +) +``` + +Then, `azure.ai.agentserver.agentframework` package provides way to run above agent as an http server and receives user input direct from http request: + +```text +# requirements.txt +# pin version to avoid breaking changes or compatibility issues +azure-ai-agentserver-agentframework==1.0.0b10 +azure-ai-agentserver-core==1.0.0b10 +``` + +```python +from azure.ai.agentserver.agentframework import from_agent_framework + +# async method +await from_agent_framework(agent).run_async() + +# or, sync method +from_agent_framework(agent).run() +``` + +Notes: +- User may or may not have `azure.ai.agentserver.agentframework` installed, if not, install it via or equivalent with other package managers: + `pip install azure-ai-agentserver-core==1.0.0b10 azure-ai-agentserver-agentframework==1.0.0b10` + +- When changing the startup command line, make sure the http server mode is the default one (without any additional flag), which is better for further development (like local debugging) and deployment (like containerization and deploy to Microsoft Foundry). + +- If loading env variables from `.env` file, like `load_dotenv()`, make sure set `override=True` to let the env variables work in deployed environment, like `load_dotenv(override=True)` + +### Request/Response Requirements + +To handle http request as user input, the workflow's starter executor should have handler to support `list[ChatMessage]` as input, like: + +```python + @handler + async def some_handler(self, messages: list[ChatMessage], ctx: WorkflowContext[...]) -> ...: +``` + +Also, to let http response returns agent output, need to add `AgentRunUpdateEvent` to context, like: + +```python + from agent_framework import AgentRunUpdateEvent, AgentRunResponseUpdate, TextContent, Role + ... + response = await self.agent.run(messages) + for message in response.messages: + if message.role == Role.ASSISTANT: + await ctx.add_event( + AgentRunUpdateEvent( + self.id, + data=AgentRunResponseUpdate( + contents=[TextContent(text=f"Agent: {message.contents[-1].text}")], + role=Role.ASSISTANT, + response_id=str(uuid4()), + ), + ) + ) +``` + +## Notes + +- This step focuses on code changes to prepare an HTTP server-based agent, not actually containerizing or deploying, thus no need to generate extra files. +- Pin `agent-framework` to version `1.0.0b260107` to avoid breaking renaming changes like `AgentRunResponseUpdate`/`AgentResponseUpdate`, `create_agent`/`as_agent`, etc. diff --git a/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/__snapshots__/triggers.test.ts.snap new file mode 100644 index 000000000..3061d9516 --- /dev/null +++ b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/__snapshots__/triggers.test.ts.snap @@ -0,0 +1,100 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`agent-framework-deploy - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` +{ + "description": "Deploy AI agents and workflows built with Microsoft Agent Framework SDK to Microsoft Foundry. Handles Dockerfile generation, ACR container build, hosted agent version creation, and container startup. +USE FOR: deploy agent to Foundry, publish agent to production, host agent on Azure, deploy workflow to Foundry, go live with agent, production deployment agent, deploy agent framework app, push agent to Azure AI Foundry, containerize agent, deploy multi-agent workflow. +DO NOT USE FOR: creating agents (use agent-framework), deploying models (use models/deploy-model), deploying web apps or functions (use azure-deploy), managing quotas (use quota). +", + "extractedKeywords": [ + "agent", + "agent-framework", + "agents", + "apps", + "azure", + "azure-deploy", + "build", + "built", + "cli", + "container", + "containerize", + "creating", + "creation", + "deploy", + "deploy-model", + "deploying", + "deployment", + "dockerfile", + "foundry", + "framework", + "functions", + "generation", + "handles", + "host", + "hosted", + "identity", + "live", + "managing", + "microsoft", + "models", + "multi-agent", + "production", + "publish", + "push", + "quota", + "quotas", + "startup", + "version", + "with", + "workflow", + "workflows", + ], + "name": "agent-framework-deploy", +} +`; + +exports[`agent-framework-deploy - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` +[ + "agent", + "agent-framework", + "agents", + "apps", + "azure", + "azure-deploy", + "build", + "built", + "cli", + "container", + "containerize", + "creating", + "creation", + "deploy", + "deploy-model", + "deploying", + "deployment", + "dockerfile", + "foundry", + "framework", + "functions", + "generation", + "handles", + "host", + "hosted", + "identity", + "live", + "managing", + "microsoft", + "models", + "multi-agent", + "production", + "publish", + "push", + "quota", + "quotas", + "startup", + "version", + "with", + "workflow", + "workflows", +] +`; diff --git a/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/integration.test.ts b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/integration.test.ts new file mode 100644 index 000000000..7261e7aec --- /dev/null +++ b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/integration.test.ts @@ -0,0 +1,99 @@ +/** + * Integration Tests for agent-framework-deploy + * + * Tests skill behavior with a real Copilot agent session. + * Runs prompts multiple times to measure skill invocation rate. + * + * Prerequisites: + * 1. npm install -g @github/copilot-cli + * 2. Run `copilot` and authenticate + */ + +import * as fs from "fs"; +import { + useAgentRunner, + AgentMetadata, + isSkillInvoked, + getToolCalls, + shouldSkipIntegrationTests, + getIntegrationSkipReason, +} from "../../../../utils/agent-runner"; + +const SKILL_NAME = "microsoft-foundry"; +const RUNS_PER_PROMPT = 5; +const EXPECTED_INVOCATION_RATE = 0.6; + +/** Terminate on first `create` tool call to avoid unnecessary file writes. */ +function terminateOnCreate(metadata: AgentMetadata): boolean { + return getToolCalls(metadata, "create").length > 0; +} + +const skipTests = shouldSkipIntegrationTests(); +const skipReason = getIntegrationSkipReason(); + +if (skipTests && skipReason) { + console.log(`⏭️ Skipping integration tests: ${skipReason}`); +} + +const describeIntegration = skipTests ? describe.skip : describe; + +describeIntegration("agent-framework-deploy - Integration Tests", () => { + const agent = useAgentRunner(); + describe("skill-invocation", () => { + test("invokes skill for agent deployment prompt", async () => { + let successCount = 0; + + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "Deploy my foundry agent built with Microsoft Agent Framework to Azure AI Foundry.", + shouldEarlyTerminate: terminateOnCreate, + }); + + if (isSkillInvoked(agentMetadata, SKILL_NAME)) { + successCount++; + } + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("⏭️ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + + const invocationRate = successCount / RUNS_PER_PROMPT; + console.log(`agent-framework-deploy invocation rate for agent deployment: ${(invocationRate * 100).toFixed(1)}% (${successCount}/${RUNS_PER_PROMPT})`); + fs.appendFileSync("./result-agent-framework-deploy.txt", `agent-framework-deploy invocation rate for agent deployment: ${(invocationRate * 100).toFixed(1)}% (${successCount}/${RUNS_PER_PROMPT})\n`); + expect(invocationRate).toBeGreaterThanOrEqual(EXPECTED_INVOCATION_RATE); + }); + + test("invokes skill for production deployment prompt", async () => { + let successCount = 0; + + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "Publish my agent to production on Azure AI Foundry with container deployment.", + shouldEarlyTerminate: terminateOnCreate, + }); + + if (isSkillInvoked(agentMetadata, SKILL_NAME)) { + successCount++; + } + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("⏭️ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + + const invocationRate = successCount / RUNS_PER_PROMPT; + console.log(`agent-framework-deploy invocation rate for production deployment: ${(invocationRate * 100).toFixed(1)}% (${successCount}/${RUNS_PER_PROMPT})`); + fs.appendFileSync("./result-agent-framework-deploy.txt", `agent-framework-deploy invocation rate for production deployment: ${(invocationRate * 100).toFixed(1)}% (${successCount}/${RUNS_PER_PROMPT})\n`); + expect(invocationRate).toBeGreaterThanOrEqual(EXPECTED_INVOCATION_RATE); + }); + }); +}); diff --git a/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/triggers.test.ts b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/triggers.test.ts new file mode 100644 index 000000000..26c955d84 --- /dev/null +++ b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/triggers.test.ts @@ -0,0 +1,102 @@ +/** + * Trigger Tests for agent-framework-deploy + * + * Tests that verify the skill triggers on appropriate prompts + * and does NOT trigger on unrelated prompts. + */ + +import { TriggerMatcher } from "../../../../utils/trigger-matcher"; +import { loadSkill, LoadedSkill } from "../../../../utils/skill-loader"; + +const SKILL_NAME = "microsoft-foundry/agent/deploy/agent-framework-deploy"; + +describe("agent-framework-deploy - Trigger Tests", () => { + let triggerMatcher: TriggerMatcher; + let skill: LoadedSkill; + + beforeAll(async () => { + skill = await loadSkill(SKILL_NAME); + triggerMatcher = new TriggerMatcher(skill); + }); + + describe("Should Trigger", () => { + const shouldTriggerPrompts: string[] = [ + "Deploy my agent to Azure AI Foundry", + "Deploy my agent to production on Foundry", + "Host my agent on Azure", + "Deploy my workflow to Foundry", + "Go live with my agent", + "Deploy agent to production", + "Push my agent to Azure AI Foundry", + "Deploy my multi-agent workflow", + "Production deployment for my agent", + "Deploy agent framework app to Foundry", + ]; + + test.each(shouldTriggerPrompts)( + 'triggers on: "%s"', + (prompt) => { + const result = triggerMatcher.shouldTrigger(prompt); + expect(result.triggered).toBe(true); + expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); + } + ); + }); + + describe("Should NOT Trigger", () => { + const shouldNotTriggerPrompts: string[] = [ + "What is the weather today?", + "Help me write a poem", + "Explain quantum computing", + "Help me with AWS SageMaker", + "Configure my PostgreSQL database", + "Optimize my Azure spending and reduce costs", + "Check model capacity across regions", + "Create a React web application", + "Help me with Kubernetes pods", + "Set up a virtual network in Azure", + "How do I write Python code?", + ]; + + test.each(shouldNotTriggerPrompts)( + 'does not trigger on: "%s"', + (prompt) => { + const result = triggerMatcher.shouldTrigger(prompt); + expect(result.triggered).toBe(false); + } + ); + }); + + describe("Trigger Keywords Snapshot", () => { + test("skill keywords match snapshot", () => { + expect(triggerMatcher.getKeywords()).toMatchSnapshot(); + }); + + test("skill description triggers match snapshot", () => { + expect({ + name: skill.metadata.name, + description: skill.metadata.description, + extractedKeywords: triggerMatcher.getKeywords() + }).toMatchSnapshot(); + }); + }); + + describe("Edge Cases", () => { + test("handles empty prompt", () => { + const result = triggerMatcher.shouldTrigger(""); + expect(result.triggered).toBe(false); + }); + + test("handles very long prompt", () => { + const longPrompt = "deploy agent ".repeat(100); + const result = triggerMatcher.shouldTrigger(longPrompt); + expect(typeof result.triggered).toBe("boolean"); + }); + + test("is case insensitive", () => { + const result1 = triggerMatcher.shouldTrigger("DEPLOY AGENT TO FOUNDRY"); + const result2 = triggerMatcher.shouldTrigger("deploy agent to foundry"); + expect(result1.triggered).toBe(result2.triggered); + }); + }); +}); diff --git a/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/unit.test.ts b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/unit.test.ts new file mode 100644 index 000000000..8ccbece4b --- /dev/null +++ b/tests/microsoft-foundry/agent/deploy/agent-framework-deploy/unit.test.ts @@ -0,0 +1,75 @@ +/** + * Unit Tests for agent-framework-deploy + * + * Test isolated skill logic and validation rules. + */ + +import { loadSkill, LoadedSkill } from "../../../../utils/skill-loader"; + +const SKILL_NAME = "microsoft-foundry/agent/deploy/agent-framework-deploy"; + +describe("agent-framework-deploy - Unit Tests", () => { + let skill: LoadedSkill; + + beforeAll(async () => { + skill = await loadSkill(SKILL_NAME); + }); + + describe("Skill Metadata", () => { + test("has valid SKILL.md with required fields", () => { + expect(skill.metadata).toBeDefined(); + expect(skill.metadata.name).toBe("agent-framework-deploy"); + expect(skill.metadata.description).toBeDefined(); + expect(skill.metadata.description.length).toBeGreaterThan(10); + }); + + test("description is appropriately sized", () => { + expect(skill.metadata.description.length).toBeGreaterThan(50); + expect(skill.metadata.description.length).toBeLessThan(1024); + }); + }); + + describe("Skill Content", () => { + test("has substantive content", () => { + expect(skill.content).toBeDefined(); + expect(skill.content.length).toBeGreaterThan(100); + }); + + test("contains deployment workflow section", () => { + expect(skill.content).toContain("Deployment Workflow"); + }); + + test("contains prerequisites section", () => { + expect(skill.content).toContain("Prerequisites"); + }); + + test("documents Dockerfile generation", () => { + expect(skill.content).toContain("Dockerfile"); + }); + + test("documents ACR configuration", () => { + expect(skill.content).toContain("ACR"); + expect(skill.content).toContain("az acr build"); + }); + + test("documents agent version deployment", () => { + expect(skill.content).toContain("Deploy Agent Version"); + }); + + test("documents deployment validation", () => { + expect(skill.content).toContain("Validate Deployment"); + }); + + test("references agent-as-server pattern", () => { + expect(skill.content).toContain("agent-as-server.md"); + }); + + test("specifies API version", () => { + expect(skill.content).toContain("api-version=2025-05-15-preview"); + }); + + test("documents post-deployment steps", () => { + expect(skill.content).toContain("Post-Deployment"); + }); + }); +});