From 38b7862f2230629e0fb60a8fb7af56d9bfb27968 Mon Sep 17 00:00:00 2001 From: Abigail Hartman Date: Fri, 28 Jun 2024 15:29:31 -0700 Subject: [PATCH] Enable integration tests on the PR gate (#967) --- .github/workflows/python-app.yml | 24 ++++- tests/conftest.py | 14 +++ tests/integration_tests/conftest.py | 47 ++++++++- .../dotenv_templates/dotenv.jinja2 | 96 +++++++++---------- tests/integration_tests/test_datasources.py | 26 ++--- .../integration_tests/test_startup_scripts.py | 40 -------- 6 files changed, 140 insertions(+), 107 deletions(-) delete mode 100644 tests/integration_tests/test_startup_scripts.py diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index e0a51c9be9..5895601d5d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -27,10 +27,28 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt - name: Test with pytest + env: + AZURE_OPENAI_ENDPOINT: ${{ secrets.AZUREOPENAIENDPOINT }} + AZURE_OPENAI_MODEL: ${{ secrets.AZUREOPENAIMODEL }} + AZURE_OPENAI_KEY: ${{ secrets.AZUREOPENAIKEY }} + AZURE_OPENAI_EMBEDDING_NAME: ${{ secrets.AZUREOPENAIEMBEDDINGNAME }} + AZURE_COSMOSDB_ACCOUNT: ${{ secrets.AZURECOSMOSDBACCOUNT }} + AZURE_COSMOSDB_DATABASE: ${{ secrets.AZURECOSMOSDBDATABASE }} + AZURE_COSMOSDB_CONVERSATIONS_CONTAINER: ${{ secrets.AZURECOSMOSDBCONVERSATIONSCONTAINER }} + AZURE_COSMOSDB_ACCOUNT_KEY: ${{ secrets.AZURECOSMOSDBACCOUNTKEY }} + AZURE_SEARCH_SERVICE: ${{ secrets.AZURESEARCHSERVICE }} + AZURE_SEARCH_INDEX: ${{ secrets.AZURESEARCHINDEX }} + AZURE_SEARCH_KEY: ${{ secrets.AZURESEARCHKEY }} + AZURE_SEARCH_QUERY: ${{ secrets.AZURESEARCHQUERY }} + ELASTICSEARCH_EMBEDDING_MODEL_ID: ${{ secrets.ELASTICSEARCHEMBEDDINGMODELID }} + ELASTICSEARCH_ENCODED_API_KEY: ${{ secrets.ELASTICSEARCHENCODEDAPIKEY }} + ELASTICSEARCH_ENDPOINT: ${{ secrets.ELASTICSEARCHENDPOINT }} + ELASTICSEARCH_INDEX: ${{ secrets.ELASTICSEARCHINDEX }} + ELASTICSEARCH_QUERY: ${{ secrets.ELASTICSEARCHQUERY }} run: | export PYTHONPATH=$(pwd) - coverage run --omit=tests/integration_tests -m pytest -v --show-capture=stdout -k "not integration" - coverage report -m + coverage run -m pytest -v --show-capture=stdout + coverage report -m --include=app.py,backend/*,tests/* coverage xml - name: Code Coverage Report @@ -44,7 +62,7 @@ jobs: hide_complexity: true indicators: true output: both - thresholds: '60 80' + thresholds: '50 80' test_windows: runs-on: diff --git a/tests/conftest.py b/tests/conftest.py index e69de29bb2..809dc769a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--use-keyvault-secrets", + help='Get secrets from a keyvault instead of the environment.', + action='store_true', default=False +) + + +@pytest.fixture(scope="session") +def use_keyvault_secrets(request) -> str: + return request.config.getoption("use_keyvault_secrets") \ No newline at end of file diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index bd45657d0c..3ce2db7989 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -3,6 +3,7 @@ import pytest from azure.identity import AzureCliCredential from azure.keyvault.secrets import SecretClient +from pydantic.alias_generators import to_snake VAULT_NAME = os.environ.get("VAULT_NAME") @@ -17,12 +18,52 @@ def secret_client() -> SecretClient: @pytest.fixture(scope="module") -def dotenv_template_params(secret_client: SecretClient) -> dict[str, str]: +def dotenv_template_params_from_kv(secret_client: SecretClient) -> dict[str, str]: secrets_properties_list = secret_client.list_properties_of_secrets() secrets = {} for secret in secrets_properties_list: - secrets[secret.name] = secret_client.get_secret(secret.name).value - + secret_name = to_snake(secret.name).upper() + secrets[secret_name] = secret_client.get_secret(secret.name).value + return secrets +@pytest.fixture(scope="module") +def dotenv_template_params_from_env() -> dict[str, str]: + def get_and_unset_variable(var_name): + # we need this function to ensure that the environment is clean before + # testing with generated dotenv files. + var_value = os.getenv(var_name) + os.environ[var_name] = "" + return var_value + + env_secrets = [ + "AZURE_COSMOSDB_ACCOUNT", + "AZURE_COSMOSDB_ACCOUNT_KEY", + "AZURE_COSMOSDB_CONVERSATIONS_CONTAINER", + "AZURE_COSMOSDB_DATABASE", + "AZURE_OPENAI_EMBEDDING_NAME" + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_MODEL", + "AZURE_OPENAI_KEY", + "AZURE_SEARCH_INDEX", + "AZURE_SEARCH_KEY", + "AZURE_SEARCH_QUERY", + "AZURE_SEARCH_SERVICE", + "ELASTICSEARCH_EMBEDDING_MODEL_ID", + "ELASTICSEARCH_ENCODED_API_KEY", + "ELASTICSEARCH_ENDPOINT", + "ELASTICSEARCH_INDEX", + "ELASTICSEARCH_QUERY" + ] + + return {s: get_and_unset_variable(s) for s in env_secrets} + + +@pytest.fixture(scope="module") +def dotenv_template_params(request, use_keyvault_secrets): + if use_keyvault_secrets: + return request.getfixturevalue("dotenv_template_params_from_kv") + + return request.getfixturevalue("dotenv_template_params_from_env") + diff --git a/tests/integration_tests/dotenv_templates/dotenv.jinja2 b/tests/integration_tests/dotenv_templates/dotenv.jinja2 index 9d0589059d..5aaecc3a7a 100644 --- a/tests/integration_tests/dotenv_templates/dotenv.jinja2 +++ b/tests/integration_tests/dotenv_templates/dotenv.jinja2 @@ -1,79 +1,79 @@ -DATASOURCE_TYPE={{ datasourceType }} -AZURE_OPENAI_ENDPOINT={{ azureOpenaiEndpoint }} -AZURE_OPENAI_MODEL={{ azureOpenaiModel }} -AZURE_OPENAI_KEY={{ azureOpenaiKey }} +DATASOURCE_TYPE={{ DATASOURCE_TYPE }} +AZURE_OPENAI_ENDPOINT={{ AZURE_OPENAI_ENDPOINT }} +AZURE_OPENAI_MODEL={{ AZURE_OPENAI_MODEL }} +AZURE_OPENAI_KEY={{ AZURE_OPENAI_KEY }} AZURE_OPENAI_TEMPERATURE=0 AZURE_OPENAI_TOP_P=1.0 AZURE_OPENAI_MAX_TOKENS=1000 AZURE_OPENAI_STOP_SEQUENCE= AZURE_OPENAI_SYSTEM_MESSAGE=You are an AI assistant that helps people find information. AZURE_OPENAI_PREVIEW_API_VERSION=2024-05-01-preview -AZURE_OPENAI_STREAM={{ azureOpenaiStream }} -{% if useAoaiEmbeddings and azureOpenaiEmbeddingName %} -AZURE_OPENAI_EMBEDDING_NAME={{ azureOpenaiEmbeddingName }} +AZURE_OPENAI_STREAM={{ AZURE_OPENAI_STREAM }} +{% if USE_AOAI_EMBEDDINGS and AZURE_OPENAI_EMBEDDING_NAME %} +AZURE_OPENAI_EMBEDDING_NAME={{ AZURE_OPENAI_EMBEDDING_NAME }} {% endif %} -{% if useAoaiEmbeddings %} -AZURE_OPENAI_EMBEDDING_ENDPOINT={{ azureOpenaiEndpoint }}/openai/deployments/ada/embeddings??api-version=2023-03-15-preview -AZURE_OPENAI_EMBEDDING_KEY={{ azureOpenaiKey }} +{% if USE_AOAI_EMBEDDINGS %} +AZURE_OPENAI_EMBEDDING_ENDPOINT={{ AZURE_OPENAI_ENDPOINT }}/openai/deployments/ada/embeddings?api-version=2023-03-15-preview +AZURE_OPENAI_EMBEDDING_KEY={{ AZURE_OPENAI_KEY }} {% endif %} -{% if enableChatHistory %} -AZURE_COSMOSDB_ACCOUNT={{ azureCosmosdbAccount }} -AZURE_COSMOSDB_DATABASE={{ azureCosmosdbDatabase }} -AZURE_COSMOSDB_CONVERSATIONS_CONTAINER={{ azureCosmosdbConversationsContainer }} -AZURE_COSMOSDB_ACCOUNT_KEY={{ azureCosmosdbAccountKey }} -AZURE_COSMOSDB_ENABLE_FEEDBACK={{ azureCosmosdbEnableFeedback }} +{% if ENABLE_CHAT_HISTORY %} +AZURE_COSMOSDB_ACCOUNT={{ AZURE_COSMOSDB_ACCOUNT }} +AZURE_COSMOSDB_DATABASE={{ AZURE_COSMOSDB_DATABASE }} +AZURE_COSMOSDB_CONVERSATIONS_CONTAINER={{ AZURE_COSMOSDB_CONVERSATIONS_CONTAINER }} +AZURE_COSMOSDB_ACCOUNT_KEY={{ AZURE_COSMOSDB_ACCOUNT_KEY }} +AZURE_COSMOSDB_ENABLE_FEEDBACK={{ AZURE_COSMOSDB_ENABLE_FEEDBACK }} {% endif %} -{% if datasourceType == "AzureCognitiveSearch" %} -AZURE_SEARCH_SERVICE={{ azureSearchService }} -AZURE_SEARCH_INDEX={{ azureSearchIndex }} -AZURE_SEARCH_KEY={{ azureSearchKey }} +{% if DATASOURCE_TYPE == "AZURE_COGNITIVE_SEARCH" %} +AZURE_SEARCH_SERVICE={{ AZURE_SEARCH_SERVICE }} +AZURE_SEARCH_INDEX={{ AZURE_SEARCH_INDEX }} +AZURE_SEARCH_KEY={{ AZURE_SEARCH_KEY }} AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG= AZURE_SEARCH_CONTENT_COLUMNS=content AZURE_SEARCH_FILENAME_COLUMN= AZURE_SEARCH_TITLE_COLUMN= AZURE_SEARCH_URL_COLUMN= AZURE_SEARCH_VECTOR_COLUMNS=contentVector -AZURE_SEARCH_QUERY_TYPE={{ azureSearchQueryType }} -AZURE_SEARCH_PERMITTED_GROUPS_COLUMN={{ azureSearchPermittedGroupsColumn }} -{% elif datasourceType == "Elasticsearch" %} -ELASTICSEARCH_ENDPOINT={{ elasticsearchEndpoint }} -ELASTICSEARCH_ENCODED_API_KEY={{ elasticsearchEncodedApiKey }} -ELASTICSEARCH_INDEX={{ elasticsearchIndex }} -ELASTICSEARCH_QUERY_TYPE={{ elasticsearchQueryType }} +AZURE_SEARCH_QUERY_TYPE={{ AZURE_SEARCH_QUERY_TYPE }} +AZURE_SEARCH_PERMITTED_GROUPS_COLUMN={{ AZURE_SEARCH_PERMITTED_GROUPS_COLUMN }} +{% elif DATASOURCE_TYPE == "ELASTICSEARCH" %} +ELASTICSEARCH_ENDPOINT={{ ELASTICSEARCH_ENDPOINT }} +ELASTICSEARCH_ENCODED_API_KEY={{ ELASTICSEARCH_ENCODED_API_KEY }} +ELASTICSEARCH_INDEX={{ ELASTICSEARCH_INDEX }} +ELASTICSEARCH_QUERY_TYPE={{ ELASTICSEARCH_QUERY_TYPE }} ELASTICSEARCH_CONTENT_COLUMNS=text ELASTICSEARCH_FILENAME_COLUMN= ELASTICSEARCH_TITLE_COLUMN= ELASTICSEARCH_URL_COLUMN= ELASTICSEARCH_VECTOR_COLUMNS=text_embedding.predicted_value -{% if useElasticsearchEmbeddings and elasticsearchEmbeddingModelId %} -ELASTICSEARCH_EMBEDDING_MODEL_ID={{ elasticsearchEmbeddingModelId }} +{% if USE_ELASTICSEARCH_EMBEDDINGS and ELASTICSEARCH_EMBEDDING_MODEL_ID %} +ELASTICSEARCH_EMBEDDING_MODEL_ID={{ ELASTICSEARCH_EMBEDDING_MODEL_ID }} {% endif %} -{% elif datasourceType == "AzureCosmosDb" %} -AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING={{ azureCosmosdbMongoVcoreConnectionString }} -AZURE_COSMOSDB_MONGO_VCORE_DATABASE={{ azureCosmosdbMongoVcoreDatabase }} -AZURE_COSMOSDB_MONGO_VCORE_CONTAINER={{ azureCosmosdbMongoVcoreContainer }} -AZURE_COSMOSDB_MONGO_VCORE_INDEX={{ azureCosmosdbMongoVcoreIndex }} -AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS={{ azureCosmosdbMongoVcoreContentColumns }} +{% elif DATASOURCE_TYPE == "AZURE_COSMOS_DB" %} +AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING={{ AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING }} +AZURE_COSMOSDB_MONGO_VCORE_DATABASE={{ AZURE_COSMOSDB_MONGO_VCORE_DATABASE }} +AZURE_COSMOSDB_MONGO_VCORE_CONTAINER={{ AZURE_COSMOSDB_MONGO_VCORE_CONTAINER }} +AZURE_COSMOSDB_MONGO_VCORE_INDEX={{ AZURE_COSMOSDB_MONGO_VCORE_INDEX }} +AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS={{ AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS }} AZURE_COSMOSDB_MONGO_VCORE_FILENAME_COLUMN= AZURE_COSMOSDB_MONGO_VCORE_TITLE_COLUMN= AZURE_COSMOSDB_MONGO_VCORE_URL_COLUMN= -AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS={{ azureCosmosdbMongoVcoreVectorColumns }} -{% elif datasourceType == "Pinecone" %} -PINECONE_ENVIRONMENT={{ pineconeEnvironment }} -PINECONE_API_KEY={{ pineconeApiKey }} -PINECONE_INDEX_NAME={{ pineconeIndexName }} -PINECONE_CONTENT_COLUMNS={{ pineconeContentColumns }} +AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS={{ AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS }} +{% elif DATASOURCE_TYPE == "PINECONE" %} +PINECONE_ENVIRONMENT={{ PINECONE_ENVIRONMENT }} +PINECONE_API_KEY={{ PINECONE_API_KEY }} +PINECONE_INDEX_NAME={{ PINECONE_INDEX_NAME }} +PINECONE_CONTENT_COLUMNS={{ PINECONE_CONTENT_COLUMNS }} PINECONE_FILENAME_COLUMN= PINECONE_TITLE_COLUMN= PINECONE_URL_COLUMN= -PINECONE_VECTOR_COLUMNS={{ pineconeVectorColumns }} -{% elif datasourceType == "AzureMLIndex" %} -AZURE_MLINDEX_NAME={{ azureMlIndexName }} -AZURE_MLINDEX_VERSION={{ azureMlIndexVersion }} -AZURE_ML_PROJECT_RESOURCE_ID={{ azureMlProjectResourceId }} -AZURE_MLINDEX_CONTENT_COLUMNS={{ azureMlIndexContentColumns }} +PINECONE_VECTOR_COLUMNS={{ PINECONE_VECTOR_COLUMNS }} +{% elif DATASOURCE_TYPE == "AZURE_ML_INDEX" %} +AZURE_MLINDEX_NAME={{ AZURE_ML_INDEX_NAME }} +AZURE_MLINDEX_VERSION={{ AZURE_ML_INDEX_VERSION }} +AZURE_ML_PROJECT_RESOURCE_ID={{ AZURE_ML_PROJECT_RESOURCE_ID }} +AZURE_MLINDEX_CONTENT_COLUMNS={{ AZURE_ML_INDEX_CONTENT_COLUMNS }} AZURE_MLINDEX_FILENAME_COLUMN= AZURE_MLINDEX_TITLE_COLUMN= AZURE_MLINDEX_URL_COLUMN= -AZURE_MLINDEX_VECTOR_COLUMNS={{ azureMlIndexVectorColumns }} +AZURE_MLINDEX_VECTOR_COLUMNS={{ AZURE_ML_INDEX_VECTOR_COLUMNS }} {% endif %} \ No newline at end of file diff --git a/tests/integration_tests/test_datasources.py b/tests/integration_tests/test_datasources.py index 2bf3b0f681..dd17bf92c7 100644 --- a/tests/integration_tests/test_datasources.py +++ b/tests/integration_tests/test_datasources.py @@ -83,25 +83,25 @@ def dotenv_rendered_template_path( ) if datasource != "none": - dotenv_template_params["datasourceType"] = datasource + dotenv_template_params["DATASOURCE_TYPE"] = datasource if datasource != "Elasticsearch" and use_elasticsearch_embeddings: pytest.skip("Elasticsearch embeddings not supported for test.") if datasource == "Elasticsearch": - dotenv_template_params["useElasticsearchEmbeddings"] = use_elasticsearch_embeddings + dotenv_template_params["USE_ELASTICSEARCH_EMBEDDINGS"] = use_elasticsearch_embeddings - dotenv_template_params["useAoaiEmbeddings"] = use_aoai_embeddings + dotenv_template_params["USE_AOAI_EMBEDDINGS"] = use_aoai_embeddings if use_aoai_embeddings or use_elasticsearch_embeddings: - dotenv_template_params["azureSearchQueryType"] = "vector" - dotenv_template_params["elasticsearchQueryType"] = "vector" + dotenv_template_params["AZURE_SEARCH_QUERY_TYPE"] = "vector" + dotenv_template_params["ELASTICSEARCH_QUERY_TYPE"] = "vector" else: - dotenv_template_params["azureSearchQueryType"] = "simple" - dotenv_template_params["elasticsearchQueryType"] = "simple" + dotenv_template_params["AZURE_SEARCH_QUERY_TYPE"] = "simple" + dotenv_template_params["ELASTICSEARCH_QUERY_TYPE"] = "simple" - dotenv_template_params["enableChatHistory"] = enable_chat_history - dotenv_template_params["azureOpenaiStream"] = stream + dotenv_template_params["ENABLE_CHAT_HISTORY"] = enable_chat_history + dotenv_template_params["AZURE_OPENAI_STREAM"] = stream return render_template_to_tempfile( rendered_template_name, @@ -122,11 +122,11 @@ def test_app(dotenv_rendered_template_path) -> Quart: @pytest.mark.asyncio async def test_dotenv(test_app: Quart, dotenv_template_params: dict[str, str]): - if dotenv_template_params["datasourceType"] == "AzureCognitiveSearch": - message_content = dotenv_template_params["azureSearchQuery"] + if dotenv_template_params["DATASOURCE_TYPE"] == "AzureCognitiveSearch": + message_content = dotenv_template_params["AZURE_SEARCH_QUERY"] - elif dotenv_template_params["datasourceType"] == "Elasticsearch": - message_content = dotenv_template_params["elasticsearchQuery"] + elif dotenv_template_params["DATASOURCE_TYPE"] == "Elasticsearch": + message_content = dotenv_template_params["ELASTICSEARCH_QUERY"] else: message_content = "What is Contoso?" diff --git a/tests/integration_tests/test_startup_scripts.py b/tests/integration_tests/test_startup_scripts.py deleted file mode 100644 index 8aec4cdf51..0000000000 --- a/tests/integration_tests/test_startup_scripts.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import pytest -import sys - -from subprocess import Popen, TimeoutExpired -from time import sleep - - -script_base_path = os.path.dirname( - os.path.dirname( - os.path.dirname(__file__) - ) -) - -script_timeout = 240 - -@pytest.fixture(scope="function") -def script_command(): - if sys.platform.startswith("linux"): - return "./start.sh" - - else: - return "./start.cmd" - - -def test_startup_script(script_command): - stdout = None - try: - p = Popen([script_command], cwd=script_base_path) - stdout, _ = p.communicate(timeout=script_timeout) - - except TimeoutExpired: - assert isinstance(stdout, str) - assert "127.0.0.1:50505" in stdout - p.terminate() - - - - - \ No newline at end of file