Resolve merge from main

microsoft · Aug 15, 2023 · 3ed154e · 3ed154e
2 parents 3fd2c62 + 2143f05
commit 3ed154e
Show file tree

Hide file tree

Showing 55 changed files with 975 additions and 332 deletions.
diff --git a/.gitignore b/.gitignore
@@ -483,4 +483,7 @@ webapp/node_modules/
 webapp/public/.well-known*
 
 # Auto-generated solution file from Visual Studio
-webapi/CopilotChatWebApi.sln
+webapi/CopilotChatWebApi.sln
+
+# Tesseract OCR language data files
+*.traineddata
diff --git a/README.md b/README.md
@@ -40,12 +40,14 @@ You will need the following items to run the sample:
 1. Open PowerShell as an administrator.
 2. Setup your environment.
 
-   ```powershell
-   cd <path to chat-copilot>\scripts\
-   .\Install.ps1
-   ```
-
-   > NOTE: This script will install `Chocolatey`, `dotnet-7.0-sdk`, `nodejs`, and `yarn`.
+    ```powershell
+    cd <path to chat-copilot>\scripts\
+    .\Install.ps1
+    ```
+
+    > NOTE: This script will install `Chocolatey`, `dotnet-7.0-sdk`, `nodejs`, and `yarn`.
+     
+    > NOTE: If you receive an error that the script is not digitally signed or cannot execute on the system, you may need to [change the execution policy](https://learn.microsoft.com/powershell/module/microsoft.powershell.core/about/about_execution_policies?view=powershell-7.3#change-the-execution-policy) (see list of [policies](https://learn.microsoft.com/powershell/module/microsoft.powershell.core/about/about_execution_policies?view=powershell-7.3#powershell-execution-policies) and [scopes](https://learn.microsoft.com/powershell/module/microsoft.powershell.core/about/about_execution_policies?view=powershell-7.3#execution-policy-scope)) or [unblock the script](https://learn.microsoft.com/powershell/module/microsoft.powershell.security/get-executionpolicy?view=powershell-7.3#example-4-unblock-a-script-to-run-it-without-changing-the-execution-policy).
 
 3. Configure Chat Copilot.
 

diff --git a/scripts/deploy/deploy-azure.ps1 b/scripts/deploy/deploy-azure.ps1
@@ -44,11 +44,15 @@ param(
     # SKU for the Azure App Service plan
     $WebAppServiceSku = "B1",
 
-    [ValidateSet("Volatile", "AzureCognitiveSearch", "Qdrant")]
+    [ValidateSet("Volatile", "AzureCognitiveSearch", "Qdrant", "Postgres")]
     [string]
     # What method to use to persist embeddings
     $MemoryStore = "AzureCognitiveSearch",
 
+    [SecureString]
+    # Password for the Postgres database
+    $SqlAdminPassword = "",
+
     [switch]
     # Don't deploy Cosmos DB for chat storage - Use volatile memory instead
     $NoCosmosDb,
@@ -87,6 +91,11 @@ if ($AIService -eq "OpenAI" -and !$AIApiKey) {
     exit 1
 }
 
+if ($MemoryStore -eq "Postgres" -and !$SqlAdminPassword) {
+    Write-Host "When MemoryStore is Postgres, SqlAdminPassword must be set"
+    exit 1
+}
+
 $jsonConfig = "
 {
     `\`"webAppServiceSku`\`": { `\`"value`\`": `\`"$WebAppServiceSku`\`" },
@@ -97,7 +106,8 @@ $jsonConfig = "
     `\`"deployNewAzureOpenAI`\`": { `\`"value`\`": $(If ($DeployAzureOpenAI) {"true"} Else {"false"}) },
     `\`"memoryStore`\`": { `\`"value`\`": `\`"$MemoryStore`\`" },
     `\`"deployCosmosDB`\`": { `\`"value`\`": $(If (!($NoCosmosDb)) {"true"} Else {"false"}) },
-    `\`"deploySpeechServices`\`": { `\`"value`\`": $(If (!($NoSpeechServices)) {"true"} Else {"false"}) }
+    `\`"deploySpeechServices`\`": { `\`"value`\`": $(If (!($NoSpeechServices)) {"true"} Else {"false"}) },
+    `\`"sqlAdminPassword`\`": { `\`"value`\`": `\`"$(ConvertFrom-SecureString $SqlAdminPassword -AsPlainText)`\`" }
 }
 "
 

diff --git a/scripts/deploy/deploy-azure.sh b/scripts/deploy/deploy-azure.sh
@@ -5,21 +5,22 @@
 set -e
 
 usage() {
-    echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION -ai AI_SERVICE_TYPE -aikey AI_SERVICE_KEY [OPTIONS]"
+    echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION -ai AI_SERVICE_TYPE [OPTIONS]"
     echo ""
     echo "Arguments:"
     echo "  -d, --deployment-name DEPLOYMENT_NAME      Name for the deployment (mandatory)"
     echo "  -s, --subscription SUBSCRIPTION            Subscription to which to make the deployment (mandatory)"
-    echo "  -wk, --web-api-key WEB_API_KEY             The API key for the backend web service"
-    echo "  -ai, --ai-service AI_SERVICE_TYPE          Type of AI service to use (i.e., OpenAI or AzureOpenAI)"
-    echo "  -aikey, --ai-service-key AI_SERVICE_KEY    API key for existing Azure OpenAI resource or OpenAI account"
+    echo "  -ai, --ai-service AI_SERVICE_TYPE          Type of AI service to use (i.e., OpenAI or AzureOpenAI) (mandatory)"
     echo "  -aiend, --ai-endpoint AI_ENDPOINT          Endpoint for existing Azure OpenAI resource"
+    echo "  -aikey, --ai-service-key AI_SERVICE_KEY    API key for existing Azure OpenAI resource or OpenAI account"
+    echo "  -wk, --web-api-key WEB_API_KEY             The API key for the backend web service (default: auto-generated)"
     echo "  -rg, --resource-group RESOURCE_GROUP       Resource group to which to make the deployment (default: \"rg-\$DEPLOYMENT_NAME\")"
     echo "  -r, --region REGION                        Region to which to make the deployment (default: \"South Central US\")"
     echo "  -wr, --web-app-region WEB_APP_REGION       Region to deploy to the static web app into. This must be a region that supports static web apps. (default: \"West US 2\")"
     echo "  -a, --app-service-sku WEB_APP_SVC_SKU      SKU for the Azure App Service plan (default: \"B1\")"
     echo "  -ms, --memory-store                        Method to use to persist embeddings. Valid values are"
-    echo "                                             \"AzureCognitiveSearch\" (default), \"Qdrant\" and \"Volatile\""
+    echo "                                             \"AzureCognitiveSearch\" (default), \"Qdrant\", \"Postgres\" and \"Volatile\""
+    echo "  -sap, --sql-admin-password                 Password for the PostgreSQL Server admin user"
     echo "  -nc, --no-cosmos-db                        Don't deploy Cosmos DB for chat storage - Use volatile memory instead"
     echo "  -ns, --no-speech-services                  Don't deploy Speech Services to enable speech as chat input"
     echo "  -dd, --debug-deployment                    Switches on verbose template deployment output"
@@ -84,6 +85,11 @@ while [[ $# -gt 0 ]]; do
         MEMORY_STORE=="$2"
         shift
         ;;
+        -sap|--sql-admin-password)
+        SQL_ADMIN_PASSWORD="$2"
+        shift
+        shift
+        ;;
         -nc|--no-cosmos-db)
         NO_COSMOS_DB=true
         shift
@@ -152,6 +158,13 @@ if [[ "${AI_SERVICE_TYPE,,}" = "openai" ]] && [[ -z "$AI_SERVICE_KEY" ]]; then
     exit 1
 fi
 
+# If MEMORY_STORE is Postges, then SQL_ADMIN_PASSWORD is mandatory
+if [[ "${MEMORY_STORE,,}" = "postgres" ]] && [[ -z "$SQL_ADMIN_PASSWORD" ]]; then
+    echo "When --memory-store is 'Postgres', --sql-admin-password must be set."
+    usage
+    exit 1
+fi
+
 # If resource group is not set, then set it to rg-DEPLOYMENT_NAME
 if [ -z "$RESOURCE_GROUP" ]; then
     RESOURCE_GROUP="rg-${DEPLOYMENT_NAME}"
@@ -187,6 +200,7 @@ JSON_CONFIG=$(cat << EOF
     "aiEndpoint": { "value": "$([ ! -z "$AI_ENDPOINT" ] && echo "$AI_ENDPOINT")" },
     "deployNewAzureOpenAI": { "value": $([ "$NO_NEW_AZURE_OPENAI" = true ] && echo "false" || echo "true") },
     "memoryStore": { "value": "$MEMORY_STORE" },
+    "sqlAdminPassword": { "value": "$SQL_ADMIN_PASSWORD" },
     "deployCosmosDB": { "value": $([ "$NO_COSMOS_DB" = true ] && echo "false" || echo "true") },
     "deploySpeechServices": { "value": $([ "$NO_SPEECH_SERVICES" = true ] && echo "false" || echo "true") }
 }

diff --git a/scripts/deploy/deploy-webapi.sh b/scripts/deploy/deploy-webapi.sh
@@ -11,7 +11,7 @@ usage() {
     echo "  -d, --deployment-name DEPLOYMENT_NAME   Name of the deployment from a 'deploy-azure.sh' deployment (mandatory)"
     echo "  -s, --subscription SUBSCRIPTION         Subscription to which to make the deployment (mandatory)"
     echo "  -rg, --resource-group RESOURCE_GROUP    Resource group name from a 'deploy-azure.sh' deployment (mandatory)"
-    echo "  -p, --package PACKAGE_FILE_PATH         Path to the WebAPI package file from a 'package-webapi.sh' run"
+    echo "  -p, --package PACKAGE_FILE_PATH         Path to the WebAPI package file from a 'package-webapi.sh' run (default: \"./out/webapi.zip\")"
 }
 
 # Parse arguments

diff --git a/scripts/deploy/deploy-webapp.sh b/scripts/deploy/deploy-webapp.sh
@@ -7,12 +7,12 @@ set -e
 SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 usage() {
-    echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION --ai AI_SERVICE_TYPE -aikey AI_SERVICE_KEY [OPTIONS]"
+    echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION -rg RESOURCE_GROUP -a APPLICATION_ID [OPTIONS]"
     echo ""
     echo "Arguments:"
+    echo "  -d, --deployment-name DEPLOYMENT_NAME  Name of the deployment from a 'deploy-azure.sh' deployment (mandatory)"
     echo "  -s, --subscription SUBSCRIPTION        Subscription to which to make the deployment (mandatory)"
     echo "  -rg, --resource-group RESOURCE_GROUP   Resource group name from a 'deploy-azure.sh' deployment (mandatory)"
-    echo "  -d, --deployment-name DEPLOYMENT_NAME  Name of the deployment from a 'deploy-azure.sh' deployment (mandatory)"
     echo "  -a, --application-id APPLICATION_ID    Client application ID (mandatory)"
     echo "  -au, --authority                       Authority to use for client applications that are not configured as multi-tenant. Defaults to (https://login.microsoftonline.com/common) if not specified."
     echo "  -nr, --no-redirect                     Do not attempt to register redirect URIs with the client application"

diff --git a/scripts/deploy/main.bicep b/scripts/deploy/main.bicep
@@ -54,6 +54,7 @@ param deployCosmosDB bool = true
   'Volatile'
   'AzureCognitiveSearch'
   'Qdrant'
+  'Postgres'
 ])
 param memoryStore string = 'Volatile'
 
@@ -78,6 +79,10 @@ var uniqueName = '${name}-${rgIdHash}'
 @description('Name of the Azure Storage file share to create')
 var storageFileShareName = 'aciqdrantshare'
 
+@description('PostgreSQL admin password')
+@secure()
+param sqlAdminPassword string = newGuid()
+
 resource openAI 'Microsoft.CognitiveServices/accounts@2022-12-01' = if (deployNewAzureOpenAI) {
   name: 'ai-${uniqueName}'
   location: location
@@ -250,6 +255,10 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = {
         name: 'MemoryStore:AzureCognitiveSearch:Key'
         value: memoryStore == 'AzureCognitiveSearch' ? azureCognitiveSearch.listAdminKeys().primaryKey : ''
       }
+      {
+        name: 'MemoryStore:Postgres:ConnectionString'
+        value: memoryStore == 'Postgres' ? 'Host=${postgreServerGroup.properties.serverNames[0].fullyQualifiedDomainName}:5432;Username=citus;Password=${sqlAdminPassword};Database=citus' : ''
+      }
       {
         name: 'AzureSpeech:Region'
         value: location
@@ -501,6 +510,16 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' = {
           privateLinkServiceNetworkPolicies: 'Enabled'
         }
       }
+      {
+        name: 'postgresSubnet'
+        properties: {
+          addressPrefix: '10.0.3.0/24'
+          serviceEndpoints: []
+          delegations: []
+          privateEndpointNetworkPolicies: 'Disabled'
+          privateLinkServiceNetworkPolicies: 'Enabled'
+        }
+      }
     ]
   }
 }
@@ -703,6 +722,76 @@ resource memorySourcesContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDataba
   }
 }
 
+resource postgreServerGroup 'Microsoft.DBforPostgreSQL/serverGroupsv2@2022-11-08' = if (memoryStore == 'Postgres') {
+  name: 'pg-${uniqueName}'
+  location: location
+  properties: {
+    postgresqlVersion: '15'
+    administratorLoginPassword: sqlAdminPassword
+    enableHa: false
+    coordinatorVCores: 1
+    coordinatorServerEdition: 'BurstableMemoryOptimized'
+    coordinatorStorageQuotaInMb: 32768
+    nodeVCores: 4
+    nodeCount: 0
+    nodeStorageQuotaInMb: 524288
+    nodeEnablePublicIpAccess: false
+  }
+}
+
+resource postgresDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (memoryStore == 'Postgres') {
+  name: 'privatelink.postgres.cosmos.azure.com'
+  location: 'global'
+}
+
+resource postgresPrivateEndpoint 'Microsoft.Network/privateEndpoints@2023-04-01' = if (memoryStore == 'Postgres') {
+  name: 'pg-${uniqueName}-pe'
+  location: location
+  properties: {
+    subnet: {
+      id: virtualNetwork.properties.subnets[2].id
+    }
+    privateLinkServiceConnections: [
+      {
+        name: 'postgres'
+        properties: {
+          privateLinkServiceId: postgreServerGroup.id
+          groupIds: [
+            'coordinator'
+          ]
+        }
+      }
+    ]
+  }
+}
+
+resource postgresVirtualNetworkLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (memoryStore == 'Postgres') {
+  parent: postgresDNSZone
+  name: 'pg-${uniqueName}-vnl'
+  location: 'global'
+  properties: {
+    virtualNetwork: {
+      id: virtualNetwork.id
+    }
+    registrationEnabled: true
+  }
+}
+
+resource postgresPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-04-01' = if (memoryStore == 'Postgres') {
+  #disable-next-line use-parent-property
+  name: '${postgresPrivateEndpoint.name}/default'
+  properties: {
+    privateDnsZoneConfigs: [
+      {
+        name: 'postgres'
+        properties: {
+          privateDnsZoneId: postgresDNSZone.id
+        }
+      }
+    ]
+  }
+}
+
 resource speechAccount 'Microsoft.CognitiveServices/accounts@2022-12-01' = if (deploySpeechServices) {
   name: 'cog-${uniqueName}'
   location: location