From 130b7826c6af040a720754e176efacafcbd3cf1e Mon Sep 17 00:00:00 2001 From: Kevin Mas Ruiz Date: Fri, 29 Aug 2025 15:18:50 +0200 Subject: [PATCH 1/3] chore: add retries to setup in case the environment is unstable --- .../tools/atlas/accessLists.test.ts | 8 +- tests/integration/tools/atlas/atlasHelpers.ts | 73 ++++++++++++++++--- .../integration/tools/atlas/clusters.test.ts | 8 +- .../integration/tools/atlas/projects.test.ts | 6 +- 4 files changed, 73 insertions(+), 22 deletions(-) diff --git a/tests/integration/tools/atlas/accessLists.test.ts b/tests/integration/tools/atlas/accessLists.test.ts index a711f38ff..df8f435b6 100644 --- a/tests/integration/tools/atlas/accessLists.test.ts +++ b/tests/integration/tools/atlas/accessLists.test.ts @@ -1,6 +1,6 @@ -import { describeWithAtlas, withProject } from "./atlasHelpers.js"; +import { afterAllWithRetry, beforeAllWithRetry, describeWithAtlas, withProject } from "./atlasHelpers.js"; import { expectDefined, getResponseElements } from "../../helpers.js"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; import { ensureCurrentIpInAccessList } from "../../../../src/common/atlas/accessListUtils.js"; function generateRandomIp(): string { @@ -17,13 +17,13 @@ describeWithAtlas("ip access lists", (integration) => { const cidrBlocks = [generateRandomIp() + "/16", generateRandomIp() + "/24"]; const values = [...ips, ...cidrBlocks]; - beforeAll(async () => { + beforeAllWithRetry(async () => { const apiClient = integration.mcpServer().session.apiClient; const ipInfo = await apiClient.getIpInfo(); values.push(ipInfo.currentIpv4Address); }); - afterAll(async () => { + afterAllWithRetry(async () => { const apiClient = integration.mcpServer().session.apiClient; const projectId = getProjectId(); diff --git a/tests/integration/tools/atlas/atlasHelpers.ts b/tests/integration/tools/atlas/atlasHelpers.ts index 38a69291d..1c8356e85 100644 --- a/tests/integration/tools/atlas/atlasHelpers.ts +++ b/tests/integration/tools/atlas/atlasHelpers.ts @@ -4,7 +4,7 @@ import type { ApiClient } from "../../../../src/common/atlas/apiClient.js"; import type { IntegrationTest } from "../../helpers.js"; import { setupIntegrationTest, defaultTestConfig, defaultDriverOptions } from "../../helpers.js"; import type { SuiteCollector } from "vitest"; -import { afterAll, beforeAll, describe } from "vitest"; +import { afterAll, beforeEach, describe } from "vitest"; export type IntegrationTestFunction = (integration: IntegrationTest) => void; @@ -36,19 +36,13 @@ export function withProject(integration: IntegrationTest, fn: ProjectTestFunctio return describe("with project", () => { let projectId: string = ""; - beforeAll(async () => { + beforeAllWithRetry(async () => { const apiClient = integration.mcpServer().session.apiClient; - - try { - const group = await createProject(apiClient); - projectId = group.id; - } catch (error) { - console.error("Failed to create project:", error); - throw error; - } + const group = await createProject(apiClient); + projectId = group.id; }); - afterAll(async () => { + afterAllWithRetry(async () => { const apiClient = integration.mcpServer().session.apiClient; if (projectId) { // projectId may be empty if beforeAll failed. @@ -70,6 +64,63 @@ export function withProject(integration: IntegrationTest, fn: ProjectTestFunctio }); } +export function beforeAllWithRetry(fixture: () => Promise): void { + beforeEach(async () => { + const MAX_SETUP_ATTEMPTS = 10; + const SETUP_BACKOFF_MS = 10; + let lastError: Error | undefined = undefined; + + for (let attempt = 0; attempt < MAX_SETUP_ATTEMPTS; attempt++) { + try { + await fixture(); + lastError = undefined; + break; + } catch (error: unknown) { + if (error instanceof Error) { + lastError = error; + } else { + lastError = new Error(String(error)); + } + + console.error("beforeAll(attempt:", attempt, "):", error); + await new Promise((resolve) => setTimeout(resolve, SETUP_BACKOFF_MS * attempt)); + } + } + + if (lastError) { + throw lastError; + } + }); +} + +export function afterAllWithRetry(fixture: () => Promise): void { + afterAll(async () => { + const MAX_SETUP_ATTEMPTS = 10; + const SETUP_BACKOFF_MS = 10; + let lastError: Error | undefined = undefined; + + for (let attempt = 0; attempt < MAX_SETUP_ATTEMPTS; attempt++) { + try { + await fixture(); + lastError = undefined; + break; + } catch (error) { + if (error instanceof Error) { + lastError = error; + } else { + lastError = new Error(String(error)); + } + console.error("afterAll(attempt:", attempt, "):", error); + await new Promise((resolve) => setTimeout(resolve, SETUP_BACKOFF_MS * attempt)); + } + } + + if (lastError) { + throw lastError; + } + }); +} + export function parseTable(text: string): Record[] { const data = text .split("\n") diff --git a/tests/integration/tools/atlas/clusters.test.ts b/tests/integration/tools/atlas/clusters.test.ts index 9ae7aabcb..294ba8e05 100644 --- a/tests/integration/tools/atlas/clusters.test.ts +++ b/tests/integration/tools/atlas/clusters.test.ts @@ -1,8 +1,8 @@ import type { Session } from "../../../../src/common/session.js"; import { expectDefined, getResponseElements } from "../../helpers.js"; -import { describeWithAtlas, withProject, randomId } from "./atlasHelpers.js"; +import { describeWithAtlas, withProject, randomId, afterAllWithRetry, beforeAllWithRetry } from "./atlasHelpers.js"; import type { ClusterDescription20240805 } from "../../../../src/common/atlas/openapi.js"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -60,7 +60,7 @@ describeWithAtlas("clusters", (integration) => { withProject(integration, ({ getProjectId }) => { const clusterName = "ClusterTest-" + randomId; - afterAll(async () => { + afterAllWithRetry(async () => { const projectId = getProjectId(); if (projectId) { const session: Session = integration.mcpServer().session; @@ -160,7 +160,7 @@ describeWithAtlas("clusters", (integration) => { }); describe("atlas-connect-cluster", () => { - beforeAll(async () => { + beforeAllWithRetry(async () => { const projectId = getProjectId(); await waitCluster(integration.mcpServer().session, projectId, clusterName, (cluster) => { return ( diff --git a/tests/integration/tools/atlas/projects.test.ts b/tests/integration/tools/atlas/projects.test.ts index 631b00f87..ba20f8382 100644 --- a/tests/integration/tools/atlas/projects.test.ts +++ b/tests/integration/tools/atlas/projects.test.ts @@ -1,14 +1,14 @@ import { ObjectId } from "mongodb"; -import { parseTable, describeWithAtlas } from "./atlasHelpers.js"; +import { parseTable, describeWithAtlas, afterAllWithRetry } from "./atlasHelpers.js"; import { expectDefined, getDataFromUntrustedContent, getResponseElements } from "../../helpers.js"; -import { afterAll, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; const randomId = new ObjectId().toString(); describeWithAtlas("projects", (integration) => { const projName = "testProj-" + randomId; - afterAll(async () => { + afterAllWithRetry(async () => { const session = integration.mcpServer().session; const projects = await session.apiClient.listProjects(); From 078f3870a626cc174d0149d919bde5e6552de3fc Mon Sep 17 00:00:00 2001 From: Kevin Mas Ruiz Date: Fri, 29 Aug 2025 15:22:34 +0200 Subject: [PATCH 2/3] chore: beforeAll, not beforeEach --- tests/integration/tools/atlas/atlasHelpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/tools/atlas/atlasHelpers.ts b/tests/integration/tools/atlas/atlasHelpers.ts index 1c8356e85..aa1f212ac 100644 --- a/tests/integration/tools/atlas/atlasHelpers.ts +++ b/tests/integration/tools/atlas/atlasHelpers.ts @@ -4,7 +4,7 @@ import type { ApiClient } from "../../../../src/common/atlas/apiClient.js"; import type { IntegrationTest } from "../../helpers.js"; import { setupIntegrationTest, defaultTestConfig, defaultDriverOptions } from "../../helpers.js"; import type { SuiteCollector } from "vitest"; -import { afterAll, beforeEach, describe } from "vitest"; +import { beforeAll, afterAll, describe } from "vitest"; export type IntegrationTestFunction = (integration: IntegrationTest) => void; @@ -65,7 +65,7 @@ export function withProject(integration: IntegrationTest, fn: ProjectTestFunctio } export function beforeAllWithRetry(fixture: () => Promise): void { - beforeEach(async () => { + beforeAll(async () => { const MAX_SETUP_ATTEMPTS = 10; const SETUP_BACKOFF_MS = 10; let lastError: Error | undefined = undefined; From d92624e2f3c924244644b43699781dcadbfda9de Mon Sep 17 00:00:00 2001 From: Kevin Mas Ruiz Date: Fri, 29 Aug 2025 15:39:13 +0200 Subject: [PATCH 3/3] chore: add retries also to some tests --- tests/integration/tools/atlas/atlasHelpers.ts | 41 +++++++++++++++---- .../integration/tools/atlas/projects.test.ts | 13 +++--- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/tests/integration/tools/atlas/atlasHelpers.ts b/tests/integration/tools/atlas/atlasHelpers.ts index aa1f212ac..978d9198a 100644 --- a/tests/integration/tools/atlas/atlasHelpers.ts +++ b/tests/integration/tools/atlas/atlasHelpers.ts @@ -4,7 +4,7 @@ import type { ApiClient } from "../../../../src/common/atlas/apiClient.js"; import type { IntegrationTest } from "../../helpers.js"; import { setupIntegrationTest, defaultTestConfig, defaultDriverOptions } from "../../helpers.js"; import type { SuiteCollector } from "vitest"; -import { beforeAll, afterAll, describe } from "vitest"; +import { beforeAll, afterAll, describe, it } from "vitest"; export type IntegrationTestFunction = (integration: IntegrationTest) => void; @@ -64,13 +64,14 @@ export function withProject(integration: IntegrationTest, fn: ProjectTestFunctio }); } +const MAX_ATLAS_STEP_ATTEMPTS = 10; +const SETUP_BACKOFF_MS = 10; + export function beforeAllWithRetry(fixture: () => Promise): void { beforeAll(async () => { - const MAX_SETUP_ATTEMPTS = 10; - const SETUP_BACKOFF_MS = 10; let lastError: Error | undefined = undefined; - for (let attempt = 0; attempt < MAX_SETUP_ATTEMPTS; attempt++) { + for (let attempt = 0; attempt < MAX_ATLAS_STEP_ATTEMPTS; attempt++) { try { await fixture(); lastError = undefined; @@ -95,11 +96,9 @@ export function beforeAllWithRetry(fixture: () => Promise): void { export function afterAllWithRetry(fixture: () => Promise): void { afterAll(async () => { - const MAX_SETUP_ATTEMPTS = 10; - const SETUP_BACKOFF_MS = 10; let lastError: Error | undefined = undefined; - for (let attempt = 0; attempt < MAX_SETUP_ATTEMPTS; attempt++) { + for (let attempt = 0; attempt < MAX_ATLAS_STEP_ATTEMPTS; attempt++) { try { await fixture(); lastError = undefined; @@ -121,6 +120,34 @@ export function afterAllWithRetry(fixture: () => Promise): void { }); } +export function itWithRetry(name: string, test: () => Promise): void { + // complains about not having assertions, but assertions are inside the test function + // eslint-disable-next-line + it(name, async () => { + let lastError: Error | undefined = undefined; + + for (let attempt = 0; attempt < MAX_ATLAS_STEP_ATTEMPTS; attempt++) { + try { + await test(); + lastError = undefined; + break; + } catch (error) { + if (error instanceof Error) { + lastError = error; + } else { + lastError = new Error(String(error)); + } + console.error(`${name} (attempt: ${attempt}):`, error); + await new Promise((resolve) => setTimeout(resolve, SETUP_BACKOFF_MS * attempt)); + } + } + + if (lastError) { + throw lastError; + } + }); +} + export function parseTable(text: string): Record[] { const data = text .split("\n") diff --git a/tests/integration/tools/atlas/projects.test.ts b/tests/integration/tools/atlas/projects.test.ts index ba20f8382..68f22093d 100644 --- a/tests/integration/tools/atlas/projects.test.ts +++ b/tests/integration/tools/atlas/projects.test.ts @@ -1,7 +1,7 @@ import { ObjectId } from "mongodb"; -import { parseTable, describeWithAtlas, afterAllWithRetry } from "./atlasHelpers.js"; +import { parseTable, describeWithAtlas, afterAllWithRetry, itWithRetry } from "./atlasHelpers.js"; import { expectDefined, getDataFromUntrustedContent, getResponseElements } from "../../helpers.js"; -import { describe, expect, it } from "vitest"; +import { describe, expect } from "vitest"; const randomId = new ObjectId().toString(); @@ -27,7 +27,7 @@ describeWithAtlas("projects", (integration) => { }); describe("atlas-create-project", () => { - it("should have correct metadata", async () => { + itWithRetry("should have correct metadata", async () => { const { tools } = await integration.mcpClient().listTools(); const createProject = tools.find((tool) => tool.name === "atlas-create-project"); expectDefined(createProject); @@ -36,7 +36,7 @@ describeWithAtlas("projects", (integration) => { expect(createProject.inputSchema.properties).toHaveProperty("projectName"); expect(createProject.inputSchema.properties).toHaveProperty("organizationId"); }); - it("should create a project", async () => { + itWithRetry("should create a project", async () => { const response = await integration.mcpClient().callTool({ name: "atlas-create-project", arguments: { projectName: projName }, @@ -48,7 +48,7 @@ describeWithAtlas("projects", (integration) => { }); }); describe("atlas-list-projects", () => { - it("should have correct metadata", async () => { + itWithRetry("should have correct metadata", async () => { const { tools } = await integration.mcpClient().listTools(); const listProjects = tools.find((tool) => tool.name === "atlas-list-projects"); expectDefined(listProjects); @@ -57,10 +57,9 @@ describeWithAtlas("projects", (integration) => { expect(listProjects.inputSchema.properties).toHaveProperty("orgId"); }); - it("returns project names", async () => { + itWithRetry("returns project names", async () => { const response = await integration.mcpClient().callTool({ name: "atlas-list-projects", arguments: {} }); const elements = getResponseElements(response); - expect(elements).toHaveLength(2); expect(elements[0]?.text).toMatch(/Found \d+ projects/); expect(elements[1]?.text).toContain("