diff --git a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts index c1aab39..3f2583a 100644 --- a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts +++ b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts @@ -7,7 +7,11 @@ const uploadPDF1 = path.resolve(__dirname, '../../test-files/tennis_tutorial.pdf const uploadPDF2 = path.resolve(__dirname, '../../test-files/Q3 24_EarningsRelease.pdf'); const question = "what is intel third-quarter 2024 revenue?"; -const keyword = "billion"; // more keywords needed +const keywords = ["billion", "$13.3", "$13.3-14.3", "above the midpoint", "guidance"]; // more keywords needed + +function containsAnyKeyword(response: string, keywords: string[]): boolean { + return keywords.some((keyword) => response.includes(keyword.replace(/\s+/g, ''))); +} async function setupResponseListener(page, apiResponse) { page.on('response', async (response) => { @@ -67,7 +71,7 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => { await page2.getByPlaceholder('Ask a question').fill(question); await page2.getByRole('button').nth(3).click(); await page2.waitForTimeout(10000); - let responseContainsKeyword = apiResponse && apiResponse.value.includes(keyword.replace(/\s+/g, '')); + let responseContainsKeyword = apiResponse && containsAnyKeyword(apiResponse.value, keywords); if (responseContainsKeyword) { throw new Error('LLM already has knowledge of this guide!') } @@ -146,12 +150,28 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => { await page2.waitForTimeout(10000); // Chat Attempt 2 + console.log ('Chat Attempt 2-------------------'); await page2.getByPlaceholder('Ask a question').fill(question); await page2.getByRole('button').nth(3).click(); await page2.waitForTimeout(10000); - responseContainsKeyword = apiResponse && apiResponse.value.includes(keyword.replace(/\s+/g, '')); + responseContainsKeyword = apiResponse && containsAnyKeyword(apiResponse.value, keywords); + console.log ('response:', apiResponse.value); if (!responseContainsKeyword) { - throw new Error('RAG failed') + console.log('First attempt failed. Asking a follow-up question...'); + apiResponse.value = ""; // Clear the response listener buffer + + // Ask another question + const followUpQuestion = "How is Intel performing in Q3 2024?"; + await page2.getByPlaceholder('Ask a question').fill(followUpQuestion); + await page2.getByRole('button').nth(3).click(); + await page2.waitForTimeout(10000); + + responseContainsKeyword = apiResponse && containsAnyKeyword(apiResponse.value, keywords); + console.log ('response:', apiResponse.value); + + if (!responseContainsKeyword) { + throw new Error('RAG failed after follow-up question'); + } } // Delete 1 document + Check data sources successfully deduct 1 or not