Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/on-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -273,8 +273,8 @@ jobs:
# secrets — even after a maintainer approves the run — so the EE branch,
# which checks out the private enterprise submodule using
# ${{ secrets.SUBMODULES_TOKEN }}, fails immediately with
# "Input required and not supplied: token". Fall back to the non-EE e2e
# suite (Novu V1) for fork PRs so the change still gets validated.
# "Input required and not supplied: token". Fall back to the community
# edition Novu V2 e2e suite for fork PRs so the change still gets validated.
ee: ${{ github.event.pull_request.head.repo.fork != true }}
secrets: inherit

Expand Down
10 changes: 7 additions & 3 deletions .github/workflows/reusable-api-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(inputs.ee && '[{"shard":1,"total":4},{"shard":2,"total":4},{"shard":3,"total":4},{"shard":4,"total":4}]' || '[{"shard":1,"total":1}]') }}
include: ${{ fromJson('[{"shard":1,"total":4},{"shard":2,"total":4},{"shard":3,"total":4},{"shard":4,"total":4}]') }}
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 15
permissions:
Expand Down Expand Up @@ -67,11 +67,15 @@ jobs:
shell: bash
run: wait-on --timeout=180000 http://127.0.0.1:1342/v1/health-check

- name: Run Novu V1 E2E tests
- name: Run Novu V2 CE E2E tests
if: ${{ !inputs.ee }}
env:
NOVU_V2_SHARD_INDEX: ${{ matrix.shard }}
NOVU_V2_TOTAL_SHARDS: ${{ matrix.total }}
NOVU_V2_MOCHA_REPORTER: dot
run: |
set -e
pnpm --filter @novu/api-service test:e2e:novu-v0
pnpm --filter @novu/api-service test:e2e:novu-v2-ce

- name: Run Novu V2 E2E tests
if: ${{ inputs.ee }}
Expand Down
1 change: 1 addition & 0 deletions apps/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
"test": "cross-env TS_NODE_PROJECT=tsconfig.spec.json TS_NODE_TRANSPILE_ONLY=true NODE_ENV=test NOVU_ENTERPRISE=true CLERK_ENABLED=true NODE_OPTIONS=--no-experimental-strip-types mocha --timeout 15000 --require ts-node/register --exit 'src/**/*.spec.ts'",
"test:e2e:novu-v0": "cross-env TS_NODE_PROJECT=tsconfig.e2e-v0.json NODE_ENV=test NODE_OPTIONS=--no-experimental-strip-types mocha --timeout 15000 --retries 3 --grep '#novu-v0' --require ts-node/register --exit --file e2e/setup.ts src/**/*.e2e{,-ee}.ts",
"test:e2e:novu-v2": "cross-env TS_NODE_PROJECT=tsconfig.spec.json NODE_ENV=test CI_EE_TEST=true CLERK_ENABLED=true NODE_OPTIONS='--max_old_space_size=8192 --no-experimental-strip-types' node scripts/run-novu-v2-e2e-shard.cjs",
"test:e2e:novu-v2-ce": "cross-env TS_NODE_PROJECT=tsconfig.spec.json NODE_ENV=test CI_EE_TEST=false CLERK_ENABLED=true NOVU_V2_CE_ONLY=true NODE_OPTIONS='--max_old_space_size=8192 --no-experimental-strip-types' node scripts/run-novu-v2-e2e-shard.cjs",
"migration": "cross-env NODE_ENV=local MIGRATION=true ts-node --transpileOnly",
"seed:clickhouse": "cross-env NODE_ENV=local ts-node --transpileOnly scripts/seed-clickhouse.ts",
"seed:triggers": "cross-env NODE_ENV=local ts-node --transpileOnly scripts/seed-triggers.ts",
Expand Down
77 changes: 60 additions & 17 deletions apps/api/scripts/run-novu-v2-e2e-shard.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,25 @@ const ROOT = path.resolve(__dirname, '..');
const DEFAULT_SHARD_INDEX = 1;
const DEFAULT_TOTAL_SHARDS = 1;
const NOVU_V2_TAG = '#novu-v2';
const TEST_FILE_PATTERN = /\.e2e(-ee)?\.ts$/;
const TEST_CASE_PATTERN = /\bit(?:\.only)?\s*\(/g;
const DEFAULT_MOCHA_REPORTER = process.env.CI ? 'dot' : 'spec';
const MOCHA_REPORTER = process.env.NOVU_V2_MOCHA_REPORTER || DEFAULT_MOCHA_REPORTER;

// CE fork PRs run with CI_EE_TEST=false, so cloud-EE-only behavior (outbound SSRF
// pinning, Stripe billing periods, RBAC permissions, translations, novu-app MCP)
// is unavailable. Exclude mixed files that contain those tests.
const CE_EXCLUDED_FILES = new Set([
'src/app/agents/e2e/active-conversations.e2e.ts',
'src/app/agents/e2e/agent-mcp-servers.e2e.ts',
'src/app/auth/e2e/permissions.guard.e2e.ts',
'src/app/bridge/e2e/sync.e2e.ts',
'src/app/environments-v1/e2e/api-key-environments-exposure.e2e.ts',
'src/app/environments-v2/e2e/environments-v2-diff.e2e.ts',
'src/app/events/e2e/trigger-event-ssrf.e2e.ts',
'src/app/organization/e2e/update-organization-settings.e2e.ts',
'src/app/workflows-v2/e2e/test-http-endpoint.e2e.ts',
]);

const MOCHA_ARGS = [
'--timeout',
'30000',
Expand Down Expand Up @@ -42,7 +56,33 @@ function readSortedEntries(dir) {
return fs.readdirSync(dir, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name));
}

function collectTestFiles(dir, files = []) {
function getCliArgs() {
return process.argv.slice(2).filter((arg) => arg !== '--');
}

function parseCeOnly() {
const args = getCliArgs();

return args.includes('--ce-only') || process.env.NOVU_V2_CE_ONLY === 'true';
}

function getTestFilePattern(ceOnly) {
return ceOnly ? /\.e2e\.ts$/ : /\.e2e(-ee)?\.ts$/;
}

function collectTestFileRoots(ceOnly) {
const roots = [path.join(ROOT, 'src')];

if (!ceOnly) {
roots.push(path.join(ROOT, 'e2e', 'enterprise'));
}

return roots;
}

function collectTestFiles(ceOnly, dir, files = []) {
const testFilePattern = getTestFilePattern(ceOnly);

if (!fs.existsSync(dir)) {
return files;
}
Expand All @@ -51,22 +91,18 @@ function collectTestFiles(dir, files = []) {
const fullPath = path.join(dir, entry.name);

if (entry.isDirectory()) {
collectTestFiles(fullPath, files);
collectTestFiles(ceOnly, fullPath, files);
continue;
}

if (TEST_FILE_PATTERN.test(entry.name)) {
if (testFilePattern.test(entry.name)) {
files.push(toPosixPath(path.relative(ROOT, fullPath)));
}
}

return files;
}

function getCliArgs() {
return process.argv.slice(2).filter((arg) => arg !== '--');
}

function parseShardValue(rawValue) {
const [shardIndex, totalShards] = rawValue.split('/').map((value) => Number(value));

Expand Down Expand Up @@ -128,14 +164,19 @@ function compareWeightedFiles(left, right) {
return right.weight - left.weight || compareFileNames(left.relativePath, right.relativePath);
}

function collectWeightedFiles() {
const candidates = [
...collectTestFiles(path.join(ROOT, 'src')),
...collectTestFiles(path.join(ROOT, 'e2e', 'enterprise')),
];
function isCeExcludedFile(relativePath) {
return CE_EXCLUDED_FILES.has(relativePath);
}

function collectWeightedFiles(ceOnly) {
const candidates = collectTestFileRoots(ceOnly).flatMap((dir) => collectTestFiles(ceOnly, dir));

return candidates
.map((relativePath) => {
if (ceOnly && isCeExcludedFile(relativePath)) {
return null;
}

const source = readSource(relativePath);

if (!source.includes(NOVU_V2_TAG)) {
Expand Down Expand Up @@ -186,8 +227,9 @@ function getShard(weightedFiles, shardIndex, totalShards) {
return buildShards(weightedFiles, totalShards)[shardIndex - 1];
}

function printShardSummary(shardIndex, totalShards, shard) {
console.log(`Running Novu V2 E2E shard ${shardIndex}/${totalShards} with ${shard.files.length} files (weight ${shard.weight}).`);
function printShardSummary(shardIndex, totalShards, shard, ceOnly) {
const suiteLabel = ceOnly ? 'Novu V2 CE E2E' : 'Novu V2 E2E';
console.log(`Running ${suiteLabel} shard ${shardIndex}/${totalShards} with ${shard.files.length} files (weight ${shard.weight}).`);
}

function runMocha(filePaths) {
Expand All @@ -201,14 +243,15 @@ function runMocha(filePaths) {
function run() {
applyDefaultEnv();

const ceOnly = parseCeOnly();
const { listOnly, shardIndex, totalShards } = parseShardConfig();
const shard = getShard(collectWeightedFiles(), shardIndex, totalShards);
const shard = getShard(collectWeightedFiles(ceOnly), shardIndex, totalShards);

if (!shard || shard.files.length === 0) {
throw new Error(`No files assigned to shard ${shardIndex}/${totalShards}`);
}

printShardSummary(shardIndex, totalShards, shard);
printShardSummary(shardIndex, totalShards, shard, ceOnly);

if (listOnly) {
for (const file of shard.files) {
Expand Down
Loading