Skip to content

Commit

Permalink
Enable integration tests on the PR gate (#967)
Browse files Browse the repository at this point in the history
  • Loading branch information
abhahn committed Jun 28, 2024
1 parent ff30796 commit 38b7862
Show file tree
Hide file tree
Showing 6 changed files with 140 additions and 107 deletions.
24 changes: 21 additions & 3 deletions .github/workflows/python-app.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,28 @@ jobs:
python -m pip install --upgrade pip
pip install -r requirements-dev.txt
- name: Test with pytest
env:
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZUREOPENAIENDPOINT }}
AZURE_OPENAI_MODEL: ${{ secrets.AZUREOPENAIMODEL }}
AZURE_OPENAI_KEY: ${{ secrets.AZUREOPENAIKEY }}
AZURE_OPENAI_EMBEDDING_NAME: ${{ secrets.AZUREOPENAIEMBEDDINGNAME }}
AZURE_COSMOSDB_ACCOUNT: ${{ secrets.AZURECOSMOSDBACCOUNT }}
AZURE_COSMOSDB_DATABASE: ${{ secrets.AZURECOSMOSDBDATABASE }}
AZURE_COSMOSDB_CONVERSATIONS_CONTAINER: ${{ secrets.AZURECOSMOSDBCONVERSATIONSCONTAINER }}
AZURE_COSMOSDB_ACCOUNT_KEY: ${{ secrets.AZURECOSMOSDBACCOUNTKEY }}
AZURE_SEARCH_SERVICE: ${{ secrets.AZURESEARCHSERVICE }}
AZURE_SEARCH_INDEX: ${{ secrets.AZURESEARCHINDEX }}
AZURE_SEARCH_KEY: ${{ secrets.AZURESEARCHKEY }}
AZURE_SEARCH_QUERY: ${{ secrets.AZURESEARCHQUERY }}
ELASTICSEARCH_EMBEDDING_MODEL_ID: ${{ secrets.ELASTICSEARCHEMBEDDINGMODELID }}
ELASTICSEARCH_ENCODED_API_KEY: ${{ secrets.ELASTICSEARCHENCODEDAPIKEY }}
ELASTICSEARCH_ENDPOINT: ${{ secrets.ELASTICSEARCHENDPOINT }}
ELASTICSEARCH_INDEX: ${{ secrets.ELASTICSEARCHINDEX }}
ELASTICSEARCH_QUERY: ${{ secrets.ELASTICSEARCHQUERY }}
run: |
export PYTHONPATH=$(pwd)
coverage run --omit=tests/integration_tests -m pytest -v --show-capture=stdout -k "not integration"
coverage report -m
coverage run -m pytest -v --show-capture=stdout
coverage report -m --include=app.py,backend/*,tests/*
coverage xml
- name: Code Coverage Report
Expand All @@ -44,7 +62,7 @@ jobs:
hide_complexity: true
indicators: true
output: both
thresholds: '60 80'
thresholds: '50 80'

test_windows:
runs-on:
Expand Down
14 changes: 14 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import pytest


def pytest_addoption(parser):
parser.addoption(
"--use-keyvault-secrets",
help='Get secrets from a keyvault instead of the environment.',
action='store_true', default=False
)


@pytest.fixture(scope="session")
def use_keyvault_secrets(request) -> str:
return request.config.getoption("use_keyvault_secrets")
47 changes: 44 additions & 3 deletions tests/integration_tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import pytest
from azure.identity import AzureCliCredential
from azure.keyvault.secrets import SecretClient
from pydantic.alias_generators import to_snake


VAULT_NAME = os.environ.get("VAULT_NAME")
Expand All @@ -17,12 +18,52 @@ def secret_client() -> SecretClient:


@pytest.fixture(scope="module")
def dotenv_template_params(secret_client: SecretClient) -> dict[str, str]:
def dotenv_template_params_from_kv(secret_client: SecretClient) -> dict[str, str]:
secrets_properties_list = secret_client.list_properties_of_secrets()
secrets = {}
for secret in secrets_properties_list:
secrets[secret.name] = secret_client.get_secret(secret.name).value

secret_name = to_snake(secret.name).upper()
secrets[secret_name] = secret_client.get_secret(secret.name).value

return secrets


@pytest.fixture(scope="module")
def dotenv_template_params_from_env() -> dict[str, str]:
def get_and_unset_variable(var_name):
# we need this function to ensure that the environment is clean before
# testing with generated dotenv files.
var_value = os.getenv(var_name)
os.environ[var_name] = ""
return var_value

env_secrets = [
"AZURE_COSMOSDB_ACCOUNT",
"AZURE_COSMOSDB_ACCOUNT_KEY",
"AZURE_COSMOSDB_CONVERSATIONS_CONTAINER",
"AZURE_COSMOSDB_DATABASE",
"AZURE_OPENAI_EMBEDDING_NAME"
"AZURE_OPENAI_ENDPOINT",
"AZURE_OPENAI_MODEL",
"AZURE_OPENAI_KEY",
"AZURE_SEARCH_INDEX",
"AZURE_SEARCH_KEY",
"AZURE_SEARCH_QUERY",
"AZURE_SEARCH_SERVICE",
"ELASTICSEARCH_EMBEDDING_MODEL_ID",
"ELASTICSEARCH_ENCODED_API_KEY",
"ELASTICSEARCH_ENDPOINT",
"ELASTICSEARCH_INDEX",
"ELASTICSEARCH_QUERY"
]

return {s: get_and_unset_variable(s) for s in env_secrets}


@pytest.fixture(scope="module")
def dotenv_template_params(request, use_keyvault_secrets):
if use_keyvault_secrets:
return request.getfixturevalue("dotenv_template_params_from_kv")

return request.getfixturevalue("dotenv_template_params_from_env")

96 changes: 48 additions & 48 deletions tests/integration_tests/dotenv_templates/dotenv.jinja2
Original file line number Diff line number Diff line change
@@ -1,79 +1,79 @@
DATASOURCE_TYPE={{ datasourceType }}
AZURE_OPENAI_ENDPOINT={{ azureOpenaiEndpoint }}
AZURE_OPENAI_MODEL={{ azureOpenaiModel }}
AZURE_OPENAI_KEY={{ azureOpenaiKey }}
DATASOURCE_TYPE={{ DATASOURCE_TYPE }}
AZURE_OPENAI_ENDPOINT={{ AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_MODEL={{ AZURE_OPENAI_MODEL }}
AZURE_OPENAI_KEY={{ AZURE_OPENAI_KEY }}
AZURE_OPENAI_TEMPERATURE=0
AZURE_OPENAI_TOP_P=1.0
AZURE_OPENAI_MAX_TOKENS=1000
AZURE_OPENAI_STOP_SEQUENCE=
AZURE_OPENAI_SYSTEM_MESSAGE=You are an AI assistant that helps people find information.
AZURE_OPENAI_PREVIEW_API_VERSION=2024-05-01-preview
AZURE_OPENAI_STREAM={{ azureOpenaiStream }}
{% if useAoaiEmbeddings and azureOpenaiEmbeddingName %}
AZURE_OPENAI_EMBEDDING_NAME={{ azureOpenaiEmbeddingName }}
AZURE_OPENAI_STREAM={{ AZURE_OPENAI_STREAM }}
{% if USE_AOAI_EMBEDDINGS and AZURE_OPENAI_EMBEDDING_NAME %}
AZURE_OPENAI_EMBEDDING_NAME={{ AZURE_OPENAI_EMBEDDING_NAME }}
{% endif %}
{% if useAoaiEmbeddings %}
AZURE_OPENAI_EMBEDDING_ENDPOINT={{ azureOpenaiEndpoint }}/openai/deployments/ada/embeddings??api-version=2023-03-15-preview
AZURE_OPENAI_EMBEDDING_KEY={{ azureOpenaiKey }}
{% if USE_AOAI_EMBEDDINGS %}
AZURE_OPENAI_EMBEDDING_ENDPOINT={{ AZURE_OPENAI_ENDPOINT }}/openai/deployments/ada/embeddings?api-version=2023-03-15-preview
AZURE_OPENAI_EMBEDDING_KEY={{ AZURE_OPENAI_KEY }}
{% endif %}
{% if enableChatHistory %}
AZURE_COSMOSDB_ACCOUNT={{ azureCosmosdbAccount }}
AZURE_COSMOSDB_DATABASE={{ azureCosmosdbDatabase }}
AZURE_COSMOSDB_CONVERSATIONS_CONTAINER={{ azureCosmosdbConversationsContainer }}
AZURE_COSMOSDB_ACCOUNT_KEY={{ azureCosmosdbAccountKey }}
AZURE_COSMOSDB_ENABLE_FEEDBACK={{ azureCosmosdbEnableFeedback }}
{% if ENABLE_CHAT_HISTORY %}
AZURE_COSMOSDB_ACCOUNT={{ AZURE_COSMOSDB_ACCOUNT }}
AZURE_COSMOSDB_DATABASE={{ AZURE_COSMOSDB_DATABASE }}
AZURE_COSMOSDB_CONVERSATIONS_CONTAINER={{ AZURE_COSMOSDB_CONVERSATIONS_CONTAINER }}
AZURE_COSMOSDB_ACCOUNT_KEY={{ AZURE_COSMOSDB_ACCOUNT_KEY }}
AZURE_COSMOSDB_ENABLE_FEEDBACK={{ AZURE_COSMOSDB_ENABLE_FEEDBACK }}
{% endif %}
{% if datasourceType == "AzureCognitiveSearch" %}
AZURE_SEARCH_SERVICE={{ azureSearchService }}
AZURE_SEARCH_INDEX={{ azureSearchIndex }}
AZURE_SEARCH_KEY={{ azureSearchKey }}
{% if DATASOURCE_TYPE == "AZURE_COGNITIVE_SEARCH" %}
AZURE_SEARCH_SERVICE={{ AZURE_SEARCH_SERVICE }}
AZURE_SEARCH_INDEX={{ AZURE_SEARCH_INDEX }}
AZURE_SEARCH_KEY={{ AZURE_SEARCH_KEY }}
AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG=
AZURE_SEARCH_CONTENT_COLUMNS=content
AZURE_SEARCH_FILENAME_COLUMN=
AZURE_SEARCH_TITLE_COLUMN=
AZURE_SEARCH_URL_COLUMN=
AZURE_SEARCH_VECTOR_COLUMNS=contentVector
AZURE_SEARCH_QUERY_TYPE={{ azureSearchQueryType }}
AZURE_SEARCH_PERMITTED_GROUPS_COLUMN={{ azureSearchPermittedGroupsColumn }}
{% elif datasourceType == "Elasticsearch" %}
ELASTICSEARCH_ENDPOINT={{ elasticsearchEndpoint }}
ELASTICSEARCH_ENCODED_API_KEY={{ elasticsearchEncodedApiKey }}
ELASTICSEARCH_INDEX={{ elasticsearchIndex }}
ELASTICSEARCH_QUERY_TYPE={{ elasticsearchQueryType }}
AZURE_SEARCH_QUERY_TYPE={{ AZURE_SEARCH_QUERY_TYPE }}
AZURE_SEARCH_PERMITTED_GROUPS_COLUMN={{ AZURE_SEARCH_PERMITTED_GROUPS_COLUMN }}
{% elif DATASOURCE_TYPE == "ELASTICSEARCH" %}
ELASTICSEARCH_ENDPOINT={{ ELASTICSEARCH_ENDPOINT }}
ELASTICSEARCH_ENCODED_API_KEY={{ ELASTICSEARCH_ENCODED_API_KEY }}
ELASTICSEARCH_INDEX={{ ELASTICSEARCH_INDEX }}
ELASTICSEARCH_QUERY_TYPE={{ ELASTICSEARCH_QUERY_TYPE }}
ELASTICSEARCH_CONTENT_COLUMNS=text
ELASTICSEARCH_FILENAME_COLUMN=
ELASTICSEARCH_TITLE_COLUMN=
ELASTICSEARCH_URL_COLUMN=
ELASTICSEARCH_VECTOR_COLUMNS=text_embedding.predicted_value
{% if useElasticsearchEmbeddings and elasticsearchEmbeddingModelId %}
ELASTICSEARCH_EMBEDDING_MODEL_ID={{ elasticsearchEmbeddingModelId }}
{% if USE_ELASTICSEARCH_EMBEDDINGS and ELASTICSEARCH_EMBEDDING_MODEL_ID %}
ELASTICSEARCH_EMBEDDING_MODEL_ID={{ ELASTICSEARCH_EMBEDDING_MODEL_ID }}
{% endif %}
{% elif datasourceType == "AzureCosmosDb" %}
AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING={{ azureCosmosdbMongoVcoreConnectionString }}
AZURE_COSMOSDB_MONGO_VCORE_DATABASE={{ azureCosmosdbMongoVcoreDatabase }}
AZURE_COSMOSDB_MONGO_VCORE_CONTAINER={{ azureCosmosdbMongoVcoreContainer }}
AZURE_COSMOSDB_MONGO_VCORE_INDEX={{ azureCosmosdbMongoVcoreIndex }}
AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS={{ azureCosmosdbMongoVcoreContentColumns }}
{% elif DATASOURCE_TYPE == "AZURE_COSMOS_DB" %}
AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING={{ AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING }}
AZURE_COSMOSDB_MONGO_VCORE_DATABASE={{ AZURE_COSMOSDB_MONGO_VCORE_DATABASE }}
AZURE_COSMOSDB_MONGO_VCORE_CONTAINER={{ AZURE_COSMOSDB_MONGO_VCORE_CONTAINER }}
AZURE_COSMOSDB_MONGO_VCORE_INDEX={{ AZURE_COSMOSDB_MONGO_VCORE_INDEX }}
AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS={{ AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS }}
AZURE_COSMOSDB_MONGO_VCORE_FILENAME_COLUMN=
AZURE_COSMOSDB_MONGO_VCORE_TITLE_COLUMN=
AZURE_COSMOSDB_MONGO_VCORE_URL_COLUMN=
AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS={{ azureCosmosdbMongoVcoreVectorColumns }}
{% elif datasourceType == "Pinecone" %}
PINECONE_ENVIRONMENT={{ pineconeEnvironment }}
PINECONE_API_KEY={{ pineconeApiKey }}
PINECONE_INDEX_NAME={{ pineconeIndexName }}
PINECONE_CONTENT_COLUMNS={{ pineconeContentColumns }}
AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS={{ AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS }}
{% elif DATASOURCE_TYPE == "PINECONE" %}
PINECONE_ENVIRONMENT={{ PINECONE_ENVIRONMENT }}
PINECONE_API_KEY={{ PINECONE_API_KEY }}
PINECONE_INDEX_NAME={{ PINECONE_INDEX_NAME }}
PINECONE_CONTENT_COLUMNS={{ PINECONE_CONTENT_COLUMNS }}
PINECONE_FILENAME_COLUMN=
PINECONE_TITLE_COLUMN=
PINECONE_URL_COLUMN=
PINECONE_VECTOR_COLUMNS={{ pineconeVectorColumns }}
{% elif datasourceType == "AzureMLIndex" %}
AZURE_MLINDEX_NAME={{ azureMlIndexName }}
AZURE_MLINDEX_VERSION={{ azureMlIndexVersion }}
AZURE_ML_PROJECT_RESOURCE_ID={{ azureMlProjectResourceId }}
AZURE_MLINDEX_CONTENT_COLUMNS={{ azureMlIndexContentColumns }}
PINECONE_VECTOR_COLUMNS={{ PINECONE_VECTOR_COLUMNS }}
{% elif DATASOURCE_TYPE == "AZURE_ML_INDEX" %}
AZURE_MLINDEX_NAME={{ AZURE_ML_INDEX_NAME }}
AZURE_MLINDEX_VERSION={{ AZURE_ML_INDEX_VERSION }}
AZURE_ML_PROJECT_RESOURCE_ID={{ AZURE_ML_PROJECT_RESOURCE_ID }}
AZURE_MLINDEX_CONTENT_COLUMNS={{ AZURE_ML_INDEX_CONTENT_COLUMNS }}
AZURE_MLINDEX_FILENAME_COLUMN=
AZURE_MLINDEX_TITLE_COLUMN=
AZURE_MLINDEX_URL_COLUMN=
AZURE_MLINDEX_VECTOR_COLUMNS={{ azureMlIndexVectorColumns }}
AZURE_MLINDEX_VECTOR_COLUMNS={{ AZURE_ML_INDEX_VECTOR_COLUMNS }}
{% endif %}
26 changes: 13 additions & 13 deletions tests/integration_tests/test_datasources.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,25 +83,25 @@ def dotenv_rendered_template_path(
)

if datasource != "none":
dotenv_template_params["datasourceType"] = datasource
dotenv_template_params["DATASOURCE_TYPE"] = datasource

if datasource != "Elasticsearch" and use_elasticsearch_embeddings:
pytest.skip("Elasticsearch embeddings not supported for test.")

if datasource == "Elasticsearch":
dotenv_template_params["useElasticsearchEmbeddings"] = use_elasticsearch_embeddings
dotenv_template_params["USE_ELASTICSEARCH_EMBEDDINGS"] = use_elasticsearch_embeddings

dotenv_template_params["useAoaiEmbeddings"] = use_aoai_embeddings
dotenv_template_params["USE_AOAI_EMBEDDINGS"] = use_aoai_embeddings

if use_aoai_embeddings or use_elasticsearch_embeddings:
dotenv_template_params["azureSearchQueryType"] = "vector"
dotenv_template_params["elasticsearchQueryType"] = "vector"
dotenv_template_params["AZURE_SEARCH_QUERY_TYPE"] = "vector"
dotenv_template_params["ELASTICSEARCH_QUERY_TYPE"] = "vector"
else:
dotenv_template_params["azureSearchQueryType"] = "simple"
dotenv_template_params["elasticsearchQueryType"] = "simple"
dotenv_template_params["AZURE_SEARCH_QUERY_TYPE"] = "simple"
dotenv_template_params["ELASTICSEARCH_QUERY_TYPE"] = "simple"

dotenv_template_params["enableChatHistory"] = enable_chat_history
dotenv_template_params["azureOpenaiStream"] = stream
dotenv_template_params["ENABLE_CHAT_HISTORY"] = enable_chat_history
dotenv_template_params["AZURE_OPENAI_STREAM"] = stream

return render_template_to_tempfile(
rendered_template_name,
Expand All @@ -122,11 +122,11 @@ def test_app(dotenv_rendered_template_path) -> Quart:

@pytest.mark.asyncio
async def test_dotenv(test_app: Quart, dotenv_template_params: dict[str, str]):
if dotenv_template_params["datasourceType"] == "AzureCognitiveSearch":
message_content = dotenv_template_params["azureSearchQuery"]
if dotenv_template_params["DATASOURCE_TYPE"] == "AzureCognitiveSearch":
message_content = dotenv_template_params["AZURE_SEARCH_QUERY"]

elif dotenv_template_params["datasourceType"] == "Elasticsearch":
message_content = dotenv_template_params["elasticsearchQuery"]
elif dotenv_template_params["DATASOURCE_TYPE"] == "Elasticsearch":
message_content = dotenv_template_params["ELASTICSEARCH_QUERY"]

else:
message_content = "What is Contoso?"
Expand Down
40 changes: 0 additions & 40 deletions tests/integration_tests/test_startup_scripts.py

This file was deleted.

0 comments on commit 38b7862

Please sign in to comment.