Adding SQL server integration for private preview (#845)

Co-authored-by: Abby Hartman <[email protected]>
microsoft · May 22, 2024 · 5e91823 · 5e91823
1 parent 57aa035
commit 5e91823
Show file tree

Hide file tree

Showing 16 changed files with 7,464 additions and 258 deletions.
diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
@@ -30,5 +30,5 @@ jobs:
         cache: 'npm'
         cache-dependency-path: '**/package-lock.json'
     - run: npm ci
-    - run: npm run build --if-present
+    - run: NODE_OPTIONS=--max_old_space_size=8192 npm run build --if-present
     - run: npm run test --if-present
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -24,7 +24,9 @@
       "type": "chrome",
       "request": "launch",
       "name": "Frontend: Launch Chrome debugger",
-      "url": "http://127.0.0.1:50505"
+      "url": "http://127.0.0.1:50505",
+      "webRoot": "${workspaceFolder}/frontend",
+      "smartStep": true
     }
   ]
 }
diff --git a/app.py b/app.py
@@ -88,9 +88,12 @@ async def assets(path):
     },
     "sanitize_answer": app_settings.base_settings.sanitize_answer,
 }
+
+
 # Enable Microsoft Defender for Cloud Integration
 MS_DEFENDER_ENABLED = os.environ.get("MS_DEFENDER_ENABLED", "true").lower() == "true"
 
+
 # Initialize Azure OpenAI Client
 def init_openai_client():
     azure_openai_client = None

diff --git a/backend/settings.py b/backend/settings.py
@@ -237,6 +237,7 @@ class _AzureSearchSettings(BaseSettings, DatasourcePayloadConstructor):
         extra="ignore",
         env_ignore_empty=True
     )
+    _type: Literal["azure_search"] = PrivateAttr(default="azure_search")
     service: str = Field(exclude=True)
     endpoint_suffix: str = Field(default="search.windows.net", exclude=True)
     index: str = Field(serialization_alias="index_name")
@@ -327,7 +328,7 @@ def construct_payload_configuration(
         parameters.update(self._settings.search.model_dump(exclude_none=True, by_alias=True))
 
         return {
-            "type": "azure_search",
+            "type": self._type,
             "parameters": parameters
         }
 
@@ -342,6 +343,7 @@ class _AzureCosmosDbMongoVcoreSettings(
         extra="ignore",
         env_ignore_empty=True
     )
+    _type: Literal["azure_cosmosdb"] = PrivateAttr(default="azure_cosmosdb")
     query_type: Literal['vector'] = "vector"
     connection_string: str = Field(exclude=True)
     index: str = Field(serialization_alias="index_name")
@@ -395,7 +397,7 @@ def construct_payload_configuration(
         parameters = self.model_dump(exclude_none=True, by_alias=True)
         parameters.update(self._settings.search.model_dump(exclude_none=True, by_alias=True))
         return {
-            "type": "azure_cosmos_db",
+            "type": self._type,
             "parameters": parameters
         }
 
@@ -407,6 +409,7 @@ class _ElasticsearchSettings(BaseSettings, DatasourcePayloadConstructor):
         extra="ignore",
         env_ignore_empty=True
     )
+    _type: Literal["elasticsearch"] = PrivateAttr(default="elasticsearch")
     endpoint: str
     encoded_api_key: str = Field(exclude=True)
     index: str = Field(serialization_alias="index_name")
@@ -464,7 +467,7 @@ def construct_payload_configuration(
         parameters.update(self._settings.search.model_dump(exclude_none=True, by_alias=True))
 
         return {
-            "type": "elasticsearch",
+            "type": self._type,
             "parameters": parameters
         }
 
@@ -476,6 +479,7 @@ class _PineconeSettings(BaseSettings, DatasourcePayloadConstructor):
         extra="ignore",
         env_ignore_empty=True
     )
+    _type: Literal["pinecone"] = PrivateAttr(default="pinecone")
     environment: str
     api_key: str = Field(exclude=True)
     index_name: str
@@ -530,7 +534,7 @@ def construct_payload_configuration(
         parameters.update(self._settings.search.model_dump(exclude_none=True, by_alias=True))
 
         return {
-            "type": "pinecone",
+            "type": self._type,
             "parameters": parameters
         }
 
@@ -542,6 +546,7 @@ class _AzureMLIndexSettings(BaseSettings, DatasourcePayloadConstructor):
         extra="ignore",
         env_ignore_empty=True
     )
+    _type: Literal["azure_ml_index"] = PrivateAttr(default="azure_ml_index")
     name: str
     version: str
     project_resource_id: str = Field(validation_alias="AZURE_ML_PROJECT_RESOURCE_ID")
@@ -582,11 +587,49 @@ def construct_payload_configuration(
         parameters.update(self._settings.search.model_dump(exclude_none=True, by_alias=True))
 
         return {
-            "type": "azure_ml_index",
+            "type": self._type,
             "parameters": parameters
         }
 
 
+class _AzureSqlServerSettings(BaseSettings, DatasourcePayloadConstructor):
+    model_config = SettingsConfigDict(
+        env_prefix="AZURE_SQL_SERVER_",
+        env_file=DOTENV_PATH,
+        extra="ignore"
+    )
+    _type: Literal["azure_sql_server"] = PrivateAttr(default="azure_sql_server")
+
+    connection_string: str = Field(exclude=True)
+    table_schema: str
+    schema_max_row: Optional[int] = None
+    top_n_results: Optional[int] = None
+
+    # Constructed fields
+    authentication: Optional[dict] = None
+
+    @model_validator(mode="after")
+    def construct_authentication(self) -> Self:
+        self.authentication = {
+            "type": "connection_string",
+            "connection_string": self.connection_string
+        }
+        return self
+
+    def construct_payload_configuration(
+        self,
+        *args,
+        **kwargs
+    ):
+        parameters = self.model_dump(exclude_none=True, by_alias=True)
+        #parameters.update(self._settings.search.model_dump(exclude_none=True, by_alias=True))
+
+        return {
+            "type": self._type,
+            "parameters": parameters
+        }
+
+
 class _BaseSettings(BaseSettings):
     model_config = SettingsConfigDict(
         env_file=DOTENV_PATH,
@@ -652,6 +695,10 @@ def set_datasource_settings(self) -> Self:
         elif self.base_settings.datasource_type == "AzureMLIndex":
             self.datasource = _AzureMLIndexSettings(settings=self, _env_file=DOTENV_PATH)
             logging.debug("Using Azure ML Index")
+
+        elif self.base_settings.datasource_type == "AzureSqlServer":
+            self.datasource = _AzureSqlServerSettings(settings=self, _env_file=DOTENV_PATH)
+            logging.debug("Using SQL Server")
 
         else:
             self.datasource = None