diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa608d95..cd9bb720 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,6 @@ jobs: - name: Security Audit working-directory: apps/web - continue-on-error: true run: npm audit --audit-level=critical --omit=dev - name: Generate Prisma Client @@ -64,7 +63,7 @@ jobs: - name: Setup Database Schema working-directory: apps/web - run: npx prisma db push --config prisma/prisma.config.ts --schema prisma/schema.prisma --skip-generate + run: npx prisma migrate deploy --config prisma/prisma.config.ts --schema prisma/schema.prisma env: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_ci" DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_ci" @@ -73,6 +72,30 @@ jobs: working-directory: apps/web run: npm run typecheck + - name: Lint + working-directory: apps/web + run: npm run lint + + - name: Unit Tests + working-directory: apps/web + run: npm run test:unit + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_ci" + DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_ci" + NEXTAUTH_SECRET: ci-secret-not-real + NEXTAUTH_URL: http://localhost:3000 + REDIS_URL: "redis://localhost:6379" + + - name: Coverage Gate + working-directory: apps/web + run: npm run test:coverage + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_ci" + DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_ci" + NEXTAUTH_SECRET: ci-secret-not-real + NEXTAUTH_URL: http://localhost:3000 + REDIS_URL: "redis://localhost:6379" + - name: Build working-directory: apps/web run: npm run build @@ -90,6 +113,30 @@ jobs: name: API Strict Typecheck (apps/api) runs-on: ubuntu-22.04 + services: + postgres: + image: ankane/pgvector:latest + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: craftmyfunnel_api_ci + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v4 @@ -108,7 +155,14 @@ jobs: - name: Generate Prisma Client working-directory: apps/api - run: DATABASE_URL="postgresql://postgres:postgres@localhost:5432/dummy" DIRECT_URL="postgresql://postgres:postgres@localhost:5432/dummy" npx prisma generate --config prisma/prisma.config.ts --schema prisma/schema.prisma + run: DATABASE_URL="postgresql://postgres:postgres@localhost:5432/craftmyfunnel_api_ci" DIRECT_URL="postgresql://postgres:postgres@localhost:5432/craftmyfunnel_api_ci" npx prisma generate --config prisma/prisma.config.ts --schema prisma/schema.prisma + + - name: Setup API Database Schema + working-directory: apps/api + run: npx prisma migrate deploy --config prisma/prisma.config.ts --schema prisma/schema.prisma + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_api_ci" + DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_api_ci" - name: Type Check working-directory: apps/api @@ -118,6 +172,14 @@ jobs: working-directory: apps/api run: npm run build + - name: API Tests + working-directory: apps/api + run: npm run test + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_api_ci" + DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/craftmyfunnel_api_ci" + REDIS_URL: "redis://localhost:6379" + docker-smoke: name: Docker Build Smoke (api required, edge-fastapi optional) runs-on: ubuntu-22.04 diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 007333f5..723951ed 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -48,11 +48,11 @@ jobs: - name: Install dependencies run: npm ci --workspace apps/web --include-workspace-root --no-audit --no-fund --legacy-peer-deps - - name: Generate Prisma Client & Push Schema + - name: Generate Prisma Client & Apply Migrations working-directory: apps/web run: | npx prisma generate --config prisma/prisma.config.ts --schema prisma/schema.prisma - npx prisma db push --config prisma/prisma.config.ts --schema prisma/schema.prisma + npx prisma migrate deploy --config prisma/prisma.config.ts --schema prisma/schema.prisma env: DATABASE_URL: postgresql://postgres:password@localhost:5432/craftmyfunnel DIRECT_URL: postgresql://postgres:password@localhost:5432/craftmyfunnel @@ -71,6 +71,7 @@ jobs: DATABASE_URL: postgresql://postgres:password@localhost:5432/craftmyfunnel DIRECT_URL: postgresql://postgres:password@localhost:5432/craftmyfunnel NEXTAUTH_SECRET: ci-playwright-secret + METRICS_TOKEN: ci-metrics-token NEXTAUTH_URL: http://localhost:3000 NEXT_PUBLIC_API_URL: http://localhost:3001 REDIS_URL: redis://localhost:6379 @@ -81,12 +82,13 @@ jobs: - name: Run Playwright tests working-directory: apps/web - run: npx playwright test + run: npx playwright test e2e/auth.spec.ts e2e/dashboard.spec.ts env: DATABASE_URL: postgresql://postgres:password@localhost:5432/craftmyfunnel DIRECT_URL: postgresql://postgres:password@localhost:5432/craftmyfunnel NEXTAUTH_SECRET: ci-playwright-secret NEXTAUTH_URL: http://localhost:3000 + METRICS_TOKEN: ci-metrics-token TEST_USER_EMAIL: audit_user@example.com TEST_USER_PASSWORD: AuditPassword123! REDIS_URL: redis://localhost:6379 diff --git a/.github/workflows/production-gate.yml b/.github/workflows/production-gate.yml index 7d8750dc..7489d831 100644 --- a/.github/workflows/production-gate.yml +++ b/.github/workflows/production-gate.yml @@ -62,11 +62,31 @@ jobs: - name: Setup Database working-directory: apps/web - run: npx prisma db push --config prisma/prisma.config.ts --schema prisma/schema.prisma + run: npx prisma migrate deploy --config prisma/prisma.config.ts --schema prisma/schema.prisma env: DATABASE_URL: "postgresql://postgres:password@localhost:5432/craftmyfunnel" DIRECT_URL: "postgresql://postgres:password@localhost:5432/craftmyfunnel" + - name: Run Unit Tests + working-directory: apps/web + run: npm run test:unit + env: + DATABASE_URL: "postgresql://postgres:password@localhost:5432/craftmyfunnel" + DIRECT_URL: "postgresql://postgres:password@localhost:5432/craftmyfunnel" + NEXTAUTH_SECRET: ci-secret + NEXTAUTH_URL: http://localhost:3000 + REDIS_URL: "redis://localhost:6379" + + - name: Run Coverage Gate + working-directory: apps/web + run: npm run test:coverage + env: + DATABASE_URL: "postgresql://postgres:password@localhost:5432/craftmyfunnel" + DIRECT_URL: "postgresql://postgres:password@localhost:5432/craftmyfunnel" + NEXTAUTH_SECRET: ci-secret + NEXTAUTH_URL: http://localhost:3000 + REDIS_URL: "redis://localhost:6379" + - name: Seed Readiness Data working-directory: apps/web run: npm run readiness:seed @@ -84,6 +104,7 @@ jobs: REDIS_URL: "redis://localhost:6379" NEXTAUTH_SECRET: ci-secret NEXTAUTH_URL: http://localhost:3000 + METRICS_TOKEN: ci-metrics-token NODE_ENV: production GEMINI_API_KEY: "ci-dummy-key" diff --git a/AGENTS.md b/AGENTS.md index 77fc410d..3193c3a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,35 +2,37 @@ This repo is a monorepo with multiple deployable apps. Treat each app as a separate service. -## What runs where +## What Runs Where -- `apps/web` - Next.js web app (UI + Next route handlers) +- `apps/web` - Next.js web app (UI plus selected route handlers) - `apps/api` - Fastify API (loads Next-style `routes/**/route.ts` handlers via an adapter) - `apps/edge-fastapi` - optional private edge runtime -- Shared packages live in `packages/*` +- shared packages live in `packages/*` -## Local dev quick path +## Local Dev Quick Path - Start Postgres + Redis + Web + API: `npm run beta:start` - Start Postgres + Redis + Web + API + Edge: `npm run beta:start:all` -## Prisma + Redis expectations +## Prisma And Redis Expectations -- Postgres is required for real app functionality (auth, data, workflows). +- Postgres is required for real app functionality: auth, data, workflows, readiness checks. - Redis is optional for cache/queue features; the app should boot without Redis. -- In CI/build-only environments, avoid hard requirements on Redis/Postgres unless the workflow explicitly provisions them. +- In CI and build-only environments, avoid hard requirements on Redis/Postgres unless the workflow explicitly provisions them. ## CI / GitHub Actions -- If a workflow needs Postgres/Redis, use GitHub Actions `services:` containers and run `prisma db push` (ephemeral DB) before tests/build steps. +- If a workflow needs Postgres/Redis, use GitHub Actions `services:` containers and run the required Prisma setup before tests/build steps. - Defaulting to `redis://localhost:6379` is fine for local dev, but must not cause CI failures when Redis is not provisioned. +- Do not treat a local pass as the final truth for launch readiness until GitHub Actions is green on the target branch. -## Editing guidance +## Editing Guidance -- Keep changes scoped to the app you're touching (`apps/web` vs `apps/api`). +- Keep changes scoped to the app you are touching (`apps/web` vs `apps/api` vs `apps/edge-fastapi`). - Prefer graceful degradation for optional infra (Redis) and fail-fast for required infra (`DATABASE_URL` in production/runtime). +- Do not silently preserve stale docs if the runtime shape changed; update the matching docs in the same run when possible. -## AI Guardrail and Credit Notes +## AI Guardrail And Credit Notes - AI generation in `apps/api` should go through `src/lib/aiService.ts` so prompt guardrails and credit enforcement are applied. - Prompt-policy and size controls are centralized in `src/lib/aiInputGuardrails.ts`. @@ -48,9 +50,41 @@ This repo is a monorepo with multiple deployable apps. Treat each app as a separ - Keep credit/billing implementation separate from public positioning; do not hide billing logic by renaming data fields. - Lead status labels should match supported application statuses unless a display-only mapping is clearly safe. -# AI Agent Scope Rules for Positioning and UI Copy Changes +## Documentation Sync Rules + +When the repo is reassessed, or when runtime boundaries change, update these together: + +1. `README.md` +2. `MASTER_SYSTEM_ARCHITECTURE.md` +3. `docs/ARCHITECTURE.md` +4. `docs/README.md` +5. `docs/context/ARCHITECTURE.md` +6. `docs/context/LAUNCH_READINESS.md` +7. the latest readiness assessment document in `docs/` + +If an older doc reflects a prior architecture, either update it or clearly mark it as historical/planning-only. + +## Repo-Wide Reassessment Workflow + +For "reassess the entire app" requests: + +1. confirm deployable services and shared packages +2. verify at least one readiness/runtime command per critical service +3. check whether the current docs match the actual repo shape +4. separate API readiness from overall launch readiness +5. call out blockers with evidence, not optimism + +Minimum useful commands: + +- `npm run readiness:audit --workspace apps/api` +- `npm run lint --workspace apps/web` +- `npm run test:coverage --workspace apps/web` +- `npm run test:e2e --workspace apps/web -- e2e/auth.spec.ts e2e/dashboard.spec.ts` + +# AI Agent Scope Rules For Positioning And UI Copy Changes For copy, positioning, or UI messaging tasks: + - Use minimum context. - Search targeted files only. - Prefer small patches over broad rewrites. @@ -62,9 +96,10 @@ For copy, positioning, or UI messaging tasks: - Before editing, identify the smallest set of files needed. - After editing, report only files changed and concise summaries. -## Token Budget Rules for AI Agents +## Token Budget Rules For AI Agents For positioning, copy, UX, and UI-label tasks: + - Use minimum context. - Do not scan the full repository. - Inspect only the smallest likely set of files. @@ -77,24 +112,24 @@ For positioning, copy, UX, and UI-label tasks: - Do not use subagents or parallel agents unless explicitly requested. - Before editing, list the exact files to inspect. - After editing, report only files changed and concise summaries. -Most important rule -Use Antigravity like this: +Most important rule: -Find files → stop. -Patch one surface → stop. -Verify banned phrases → stop. +Use Antigravity like this: -Do not ask: +Find files -> stop. +Patch one surface -> stop. +Verify banned phrases -> stop. -Analyze the full app and implement the architecture changes. +Do not ask for a full-app analysis and architecture rewrite unless the user explicitly asked for that breadth. ## Multi-Agent Review Workflow -Project Goal: +Project goal: Review and improve this application for security, customer experience, usability, reliability, and performance. Rules for all review agents: + - Do not make assumptions without checking the code. - Do not change business logic unless clearly required. - Every issue must include file name, problem, impact, suggested fix, and priority: Critical / High / Medium / Low. @@ -105,11 +140,13 @@ Rules for all review agents: - Do not allow agents to overwrite each other's work; use separate branches, workspaces, or read-only report outputs when possible. Recommended initial review agents: + - Builder Agent: app structure, main flows, broken or incomplete areas, obvious bugs, and proposed files to modify before editing. - Security Review Agent: authentication, authorization, exposed routes, input validation, injection risks, XSS, CSRF, uploads, secrets, env misuse, admin routes, rate limits, and data leakage. - Customer Experience and Usability Agent: first-time customer journey, homepage clarity, value proposition, CTAs, forms, messaging, mobile flow, navigation, accessibility basics, loading, empty, and error states. - QA Agent: smoke, functional, validation, mobile, security-related, negative, and edge-case test plan, plus missing tests and critical automation gaps. Consolidation model: + - Product Manager Agent removes duplicates, prioritizes by business impact, and divides work into Must fix before launch, Should fix soon, and Good to have. - Developer-ready tasks should include owner: Codex Builder, Security, UX, QA, or Performance. diff --git a/MASTER_SYSTEM_ARCHITECTURE.md b/MASTER_SYSTEM_ARCHITECTURE.md index 184ec377..8972e1f5 100644 --- a/MASTER_SYSTEM_ARCHITECTURE.md +++ b/MASTER_SYSTEM_ARCHITECTURE.md @@ -1,264 +1,167 @@ -# CraftMyFunnel Master System Architecture (Current) +# CraftMyFunnel Master System Architecture -## Scope +## Purpose -This document describes the current architecture used for the startup launch path: +This is the repo-level source of truth for the system that actually exists today. It replaces older descriptions that referred to a separate "managed runtime" control-plane topology that is no longer the primary shape of this repository. -- email-first private beta -- Netjana buyer-signal ingest, signal-aware campaign preparation, and human-approved follow-up review -- monorepo with independently deployable apps -- edge runtime optional and private -- public positioning is constrained to workflow preparation, review, and tracking unless a stronger capability is implemented and verified +## Deployable Services ---- +CraftMyFunnel is a monorepo with **three deployable apps**: -## Deployable Apps +1. `apps/web` - public Next.js application +2. `apps/api` - public Fastify API and worker runtime +3. `apps/edge-fastapi` - optional private FastAPI edge runtime -CraftMyFunnel has 3 deployable apps under `apps/`: +Shared code lives in `packages/*`, currently including `packages/toon-core`. -1. `apps/web` (Next.js): public web app -2. `apps/api` (Fastify): public backend API -3. `apps/edge-fastapi` (FastAPI): optional private edge execution service - -Single git repo does not mean single deployment unit. It means shared code ownership with split deployment pipelines. - ---- - -## Runtime Topology - -Full GitHub-renderable Mermaid diagrams are maintained in [`docs/architecture-diagram.md`](docs/architecture-diagram.md), including layered, request-lifecycle, control-plane, data-plane, Netjana buyer-signal, email/LinkedIn channel, and Landing Agent funnel diagrams. +## Production Topology ```mermaid flowchart LR - U[User Browser] --> W[apps/web] - V[Landing Visitor] --> W - N[Netjana Buyer Signals] --> A[apps/api] - W --> A[apps/api] - A --> P[(Postgres)] - A --> R[(Redis)] - A --> J[(Jobs + Signals)] - A --> C[Review-First Email and LinkedIn Workers] - C --> X[SMTP + LinkedIn] - A --> E[apps/edge-fastapi private optional] + Browser[Authenticated browser] --> Web[apps/web] + Visitor[Public landing visitor] --> Web + Web --> API[apps/api] + Signals[Netjana and webhooks] --> API + API --> DB[(Postgres)] + API -. optional .-> Cache[(Redis)] + API --> Jobs[Workers and async handlers] + API --> Providers[LLM, SMTP, CRM, billing providers] + API -. optional private .-> Edge[apps/edge-fastapi] ``` -### Public services - -- `web` -- `api` - -### Private/internal services - -- `postgres` -- `redis` -- `edge-fastapi` (recommended private by default) - ---- +### Public surfaces -## Layered Architecture +- `apps/web` +- `apps/api` -CraftMyFunnel is organized around explicit runtime, domain, data, and control boundaries. The goal is to keep public web delivery, API ownership, persistence, and optional edge execution separate while preserving fast cross-app development in one repo. +### Private or internal surfaces -| Layer | Name | Primary owner | Notes | -| --- | --- | --- | --- | -| 0 | Actors and Channels | Product surfaces | Browser users, anonymous landing visitors, email/LinkedIn prospects, operators, extension users, webhook senders | -| 1 | Delivery and Routing | `apps/web` + `apps/api` | Next middleware, public allowlist, feature gates, API proxy, direct API ingress, CORS, rate limits | -| 2 | Web Experience | `apps/web` | Marketing pages, dashboard, setup wizard, campaign UI, Intel dashboard, Landing Agent UI, public pages | -| 3 | API Runtime Boundary | `apps/api` | Fastify server, route loader, Next-style route adapter, request/response bridge | -| 4 | Application Services | `apps/api/routes` | Route handlers grouped by campaign, setup, billing, analytics, landing-agent, intel/webhooks, extension, admin | -| 5 | Domain Modules | `apps/api/src/modules` | Campaigns, leads, landing-agent, intel/signals, knowledge, workflows, inbox, governance, settings | -| 6 | AI and Automation | `apps/api/src/modules`, workers | Prompt builders, Netjana normalize/score/match, signal-aware email composer, model gateway, guardrails, channel workers, event store | -| 7 | Data Access and Persistence | Prisma + infra | Postgres primary state, ShadowSignal/ScrapingJob/Job rows, Redis optional cache/queue, knowledge assets, audit/system events | -| 8 | External Integrations | Provider adapters | Netjana/CraftMyFunnel Intel, LLMs, SMTP, LinkedIn/browser actions, payments, CRM/enrichment | -| 9 | Optional Private Edge | `apps/edge-fastapi` | Private edge execution, browser or hardware-backed tasks | +- Postgres +- Redis +- `apps/edge-fastapi` +- provider secrets and signing keys -### Cross-Cutting Controls - -- Auth/session context flows through web middleware and API request-aware auth helpers. -- Team RBAC and workspace selection are enforced before domain-service mutation. -- Public landing endpoints are allowlisted but still rate limited and input validated. -- Netjana webhook ingest validates `x-source`, API key scope, payload shape, and optional HMAC before any signal becomes trusted context. -- Governance approval, audit logging, and guardrails sit across publish/send/automation workflows. -- Redis-backed behavior must degrade gracefully unless a workflow explicitly provisions Redis. -- AI generation enforces centralized prompt policy by surface (`CHAT`, `HELPER`, `EMAIL`, `LANDING`, `GENERIC`) before model execution. -- AI generation enforces atomic team-credit reservation and usage settlement in the runtime generation path. -- Embedding requests now use the same guarded billing and usage logging path as text generation. -- Helper/chat/email/landing request payloads now have explicit size budgets to reduce abuse and prompt stuffing. -- Legacy extension queue endpoints are authenticated, team-scoped, and claim-aware to prevent duplicate ambiguous fetch/result mutation. -- Sensitive config and governance routes now require elevated team roles, and setup status redacts provider secrets before returning team config. -- Landing HTML is sanitized before public render to reduce stored-XSS exposure on published pages. -- Agentic RAG retrieval is campaign-scoped; the known `teamId`-as-`campaignId` mismatch is fixed. -- Public copy and generated content must avoid unsupported claims about guaranteed meetings, qualified pipeline outcomes, fully autonomous execution, conversion attribution, or outcome-based billing. Safer language is preparation, support, tracking, review, and human approval. - ---- - -## AI Guardrail and Credit Path - -The active guardrail and billing contract for AI generation is documented in: - -- [`docs/AI_GUARDRAILS_AND_TOKEN_USAGE.md`](docs/AI_GUARDRAILS_AND_TOKEN_USAGE.md) - -```mermaid -sequenceDiagram - autonumber - actor User as User or Client - participant Route as API Route - participant Guard as aiInputGuardrails - participant AI as aiService - participant Credits as reserve/settle credits - participant Model as LLM Provider - participant Usage as LLMUsageLog - - User->>Route: Request AI-assisted draft, review, or tracking support - Route->>AI: Forward validated request with team context - AI->>Guard: Enforce prompt policy and size budgets - Guard-->>AI: allow or reject - AI->>Credits: reserve estimated credits - Credits-->>AI: allow or insufficient - AI->>Model: Generate response - Model-->>AI: text + token usage - AI->>Usage: persist tokens and cost - AI->>Credits: settle usage or refund difference - AI-->>Route: bounded review-ready output - Route-->>User: success or 400/401/402 -``` - ---- - -## Buyer Signal And Channel Flow - -Netjana is connected through the Intel service path: - -1. Netjana posts buyer-intent cards to `POST /webhooks/netjana-intel`. -2. `apps/api/routes/webhooks/netjana-intel/route.ts` validates source, API key, payload, and optional `x-netjana-signature`. -3. `apps/api/src/modules/intel/service/netjanaIntelService.ts` normalizes the signal, computes strength, matches company/campaign/lead context, and writes `ScrapingJob`, `ShadowSignal`, lead `marketContext`, and lead `enrichedData.netjana`. -4. Trusted signals are written into the `Netjana Intelligence` knowledge base so RAG and email composition can use grounded buyer context. -5. Hot, verified, matched signals enqueue `INTEL_FOLLOWUP_REFRESH`. -6. `apps/api/workers/handlers/intel-followup-worker.ts` generates a signal-aware email draft with `composeNodeA`, writes activity, and opens approval for review where configured. -7. Sequence actions can continue across LinkedIn and email through review-first controls: LinkedIn visit/connect/message steps use `runLinkedInAction`, and email steps can use Netjana context from `lead.enrichedData.netjana` before sending through SMTP. - -The Landing Agent is connected to campaigns and captures public lead/event data through `LandingLead` and `LandingEvent`. Its direct `BuyerIntelAdapter` currently exists as a configurable adapter stub, so direct Netjana-to-landing-copy injection is not enabled by default; the active connection is Netjana -> Intel -> Lead/Campaign/Knowledge -> campaign preparation, follow-up review, and reporting. - ---- - -## Responsibilities By App +## Architectural Boundaries ### `apps/web` -- onboarding, dashboard, marketing, pricing, setup UI -- authenticated user flows -- Netjana Intel dashboard at `/intel` -- public landing pages at `/p/[slug]` -- calls API for business operations -- beta feature gating at route/UI level +Owns: -### `apps/api` +- marketing pages +- authenticated dashboard and setup flows +- public landing-page render path +- lightweight route handlers that are appropriate to run in the web app +- API proxy and browser-facing auth/session behavior -- campaign, lead, approval, billing, analytics APIs -- Netjana webhook and Intel summary APIs -- Landing Agent APIs and public landing ingestion -- database access via Prisma -- queue/task handling -- signal-aware email and LinkedIn sequence workers -- system health and operational endpoints +Should not become the home of core business mutations that belong in the API service. -### `apps/edge-fastapi` (optional) - -- edge execution endpoints -- hardware/edge-specific operations when enabled -- kept private for beta unless explicitly required - ---- - -## Data Boundaries - -- primary system state: Postgres -- transient queue/cache: Redis -- API owns database writes for core workflows -- web should treat API as system boundary for business operations +### `apps/api` ---- +Owns: -## Monorepo And Separate Hosting +- core business APIs +- Prisma/database access +- workers and background job dispatch +- buyer-signal intake and normalization +- AI generation path, guardrails, usage logging, and credit enforcement +- governance, audit, feature flags, and operational readiness checks -### Why One Repo +This is the system-of-record backend. -- atomic cross-app changes (UI + API + schema in one PR) -- single CI policy surface (quality/security checks) -- shared scripts and dependency management -- lower coordination overhead for early-stage team +### `apps/edge-fastapi` -### How To Host Separately From One Repo +Owns only optional edge concerns: -Define one service per app with independent root paths: +- private edge execution +- hardware-adjacent or browser-adjacent workloads +- local or isolated runtime behavior when explicitly enabled -- web service root: `apps/web` -- api service root: `apps/api` -- edge service root: `apps/edge-fastapi` +It should remain optional for the main web/API launch path. -Use path-based deploy triggers: +## Core Request Flows -- `apps/web/**` -> deploy web -- `apps/api/**` -> deploy api -- `apps/edge-fastapi/**` -> deploy edge +### 1. Authenticated product flow -For shared files (`package-lock.json`, root scripts, shared schema/config), trigger deploys for impacted services. +1. User enters through `apps/web` +2. Web middleware/session checks run +3. Business operations go to `apps/api` +4. API validates team context, roles, flags, and payloads +5. API reads/writes Postgres and may queue work ---- +### 2. Public landing flow -## Environments +1. Visitor requests `/p/[slug]` from `apps/web` +2. Published landing content is rendered from stored state +3. Lead capture and event tracking flow into `apps/api` +4. Landing HTML is sanitized before public render -Use at least: +### 3. Buyer-signal flow -- `staging` -- `production` +1. External signal source posts to API webhook endpoints +2. API validates source and payload +3. Signal is normalized and written to team/campaign/lead context +4. Hot signals can enqueue review-first follow-up refresh work +5. Human review remains the safe default for outbound actions -Recommended environment variables by service: +### 4. AI generation flow -- web: `NEXT_PUBLIC_API_URL`, auth/public keys -- api: `DATABASE_URL`, `REDIS_URL`, provider secrets, edge URI (if used) -- edge: runtime/hardware variables only when edge is enabled +1. Route validates auth, team scope, and input size +2. `aiInputGuardrails` evaluates the request +3. `aiService` reserves estimated credits +4. Model provider call executes +5. usage, cost, and settlement are persisted +6. bounded output returns to the caller ---- +## Data Ownership -## Startup Beta Mode +### Required -Current launch mode prioritizes reliability: +- **Postgres**: required for auth, data, workflow state, governance, and readiness checks -- web + api + postgres required -- redis recommended (cache/queue), but the system should boot without it -- edge optional -- linked automation surfaces gated for beta where required +### Optional -Local fast path: +- **Redis**: optional for cache/queue acceleration; runtime should degrade gracefully when absent unless a workflow explicitly depends on it -```bash -npm run beta:start -``` +### Persistence rules -This command starts infra, syncs schema, and starts/reuses web and API services. +- API is the primary owner of business-state writes +- web should treat the API as the boundary for most product mutations +- edge should not introduce an alternate source of truth -All three apps locally: +## Cross-Cutting Controls -```bash -npm run beta:start:all -``` +- role-aware access control +- feature-flag gating +- audit logging +- prompt guardrails +- credit reservation and settlement +- HTML sanitization on public landing render +- readiness, health, and metrics endpoints +- optional infra degradation for Redis -This also ensures `edge-fastapi` is up on port `8000`. +## Current Readiness View ---- +### Strong -## CI And Build Environments +- API readiness audit is currently `100/100` +- web/API/edge service boundaries are understandable and documentable +- local Docker-backed Postgres and Redis support realistic beta flows +- launch-path docs now reflect the current monorepo topology -Build/test runners (GitHub Actions, preview deploys) should not assume Postgres/Redis exist unless the pipeline provisions them. +### Not Yet Broad-Launch Ready -- Workflows that need Postgres/Redis should use service containers and run `prisma db push` before executing integration steps. -- Redis-backed features should degrade gracefully when `REDIS_URL` is not set. +- web unit coverage is currently non-deterministic in this tree, with timeouts observed in health, metrics, and worker-dispatch tests +- latest local fixes still need a confirmed green GitHub Actions run +- dependency security debt remains in the npm dependency graph +- edge runtime remains optional and should not be considered part of the required launch path ---- +## Documentation Contract -## Decision Summary +When the runtime shape changes, update these files together: -There are 3 apps because responsibilities are different and deploy cadence is different. -There is 1 repo because coordination speed and consistency matter more than repo count at startup stage. -They are hosted separately by using app-level service roots and path-filtered deploy pipelines. +1. `README.md` +2. `MASTER_SYSTEM_ARCHITECTURE.md` +3. `docs/ARCHITECTURE.md` +4. `docs/context/ARCHITECTURE.md` +5. `docs/context/LAUNCH_READINESS.md` +6. the current readiness assessment in `docs/` diff --git a/README.md b/README.md index fd8b16c1..fe28f22f 100644 --- a/README.md +++ b/README.md @@ -1,83 +1,58 @@ # CraftMyFunnel -CraftMyFunnel is a multi-app monorepo for AI-assisted outbound, campaign operations, landing-page funnels, governance, and launch-readiness workflows. +CraftMyFunnel is a multi-app monorepo for AI-assisted outbound operations, landing-page funnels, buyer-signal review, and team governance. The repo ships three deployable services with shared contracts and shared operational docs. -The repository is organized as one codebase with multiple deployable services. The root is orchestration only: scripts, shared config, CI, and documentation. +## Current Status -## What This Repo Contains +As of **June 2, 2026**: -| Path | Service | Purpose | -| --- | --- | --- | -| `apps/web` | Next.js web app | Marketing pages, authenticated dashboard, setup wizard, public landing pages, and web route handlers | -| `apps/api` | Fastify API app | Core backend APIs, workers, Prisma access, auth-aware route adapter, landing-agent APIs | -| `apps/edge-fastapi` | FastAPI edge runtime | Optional private edge execution service | -| `packages/*` | Shared packages | Shared contracts, helpers, and cross-app code | -| `docs/*` | Documentation | Architecture, setup, runbooks, diagrams, and implementation notes | +- `apps/api` production-readiness audit: **100/100** +- broader launch readiness: **not fully green yet** +- biggest remaining risk: the web coverage lane is **not stable** in the current tree +- GitHub Actions still need a fresh confirmed green run after the latest local fixes + +That means the platform is in good shape for controlled beta work, but not yet at the confidence bar for broad production traffic. -## System Architecture +## Monorepo Layout -For the full GitHub-renderable Mermaid architecture, see [docs/architecture-diagram.md](./docs/architecture-diagram.md). It includes the layered system design, request lifecycle, control/data plane split, platform runtime, Netjana buyer-signal flow, email/LinkedIn channel flow, and Landing Agent funnel. +| Path | Service | Role | +| --- | --- | --- | +| `apps/web` | Next.js app | Marketing pages, authenticated product UI, public landing pages, selected route handlers | +| `apps/api` | Fastify app | Core API, workers, Prisma access, webhooks, AI runtime integration, readiness checks | +| `apps/edge-fastapi` | FastAPI edge runtime | Optional private execution service for edge or hardware-backed tasks | +| `packages/toon-core` | Shared package | Shared serialization/helpers for AI context handling | +| `docs/*` | Docs | Architecture, readiness, deployment, QA, and implementation records | + +## Runtime Topology ```mermaid flowchart LR - GitHub[GitHub Repo and Actions] --> Vercel[Vercel - apps/web Next.js] - GitHub --> RailwayAPI[Railway - apps/api Fastify] - GitHub -. optional .-> RailwayEdge[Railway Private - apps/edge-fastapi] - - Browser[Authenticated User Browser] --> Vercel - Visitor[Public Landing Visitor] --> Vercel - Netjana[Netjana Buyer Signals] --> RailwayAPI - - Vercel --> WebRoutes[Marketing, Dashboard, Setup, Landing Pages] - Vercel --> Proxy["/api/proxy/*"] - Proxy --> RailwayAPI - - RailwayAPI --> Postgres[(Managed Postgres)] - RailwayAPI -. recommended .-> Redis[(Managed Redis)] - RailwayAPI --> Signals[(Signals, Jobs, Knowledge, Audit)] - RailwayAPI --> Channels[Email and LinkedIn Workers] - RailwayAPI --> AI[OpenAI / Anthropic / Gemini] - RailwayAPI --> Billing[Razorpay] - RailwayAPI --> CRM[HubSpot / Salesforce Optional] - RailwayAPI -. private optional .-> RailwayEdge - - Channels --> GmailSMTP[Gmail / Google Workspace SMTP] - Channels -. optional .-> WhatsApp[Meta WhatsApp / Twilio] - Vercel -. errors .-> Sentry[Sentry Optional] - Cloudflare[Cloudflare DNS / WAF Optional] --> Vercel + Browser[Signed-in user] --> Web[apps/web] + Visitor[Public landing visitor] --> Web + Netjana[Buyer signals] --> API[apps/api] + Web --> API + API --> Postgres[(Postgres)] + API -. optional .-> Redis[(Redis)] + API --> Workers[Jobs, audit, signals, follow-up workers] + API --> Models[OpenAI / Anthropic / Gemini] + API -. optional private .-> Edge[apps/edge-fastapi] ``` -## Product Surfaces - -- Outreach campaigns, leads, analytics, approvals, and inbox workflows. -- Netjana Intel dashboard and webhook path for buyer-intent signals, lead/campaign matching, knowledge enrichment, and hot-signal follow-up review. -- Landing Agent funnel builder with prompt intake, brief generation, wireframes, constrained editor, public publish path, lead capture, and event tracking. -- Setup wizard for brand, email, AI, LinkedIn extension, readiness, and launch configuration. -- Governance, audit, feature gating, team settings, and approval controls. -- Optional private edge runtime for hardware or browser-backed execution. +## What The Product Actually Does Today -## Security And AI Guardrails +The safe description is: -Recent hardening is now active across AI generation and outbound surfaces: +- helps teams manage outreach campaigns, leads, approvals, landing funnels, and setup +- ingests buyer-intent signals and turns them into review-ready follow-up context +- supports AI-assisted drafting and analysis with centralized guardrails and credit enforcement +- tracks landing-page leads and events +- keeps governance, audit, and feature gating inside the runtime path -- legacy queue endpoints are auth-gated, team-scoped, and claim-aware -- prompt-injection and script payload checks are centralized in `apps/api/src/lib/aiInputGuardrails.ts` -- route-level size limits are enforced for helper/chat/email/landing generation paths -- AI generation uses atomic credit reservation + settlement in `apps/api/src/lib/aiService.ts` -- embedding requests now go through guarded billing and usage logging -- landing-page rendering now sanitizes stored HTML before public render -- setup, SMTP, policy, guardrail, key, and team-management routes are explicitly role-gated -- token usage and cost are recorded in `LLMUsageLog` and usage deductions are recorded in `CreditTransaction` - -For the full policy contract and Mermaid visual: - -- [AI guardrails and token usage](./docs/AI_GUARDRAILS_AND_TOKEN_USAGE.md) -- [Hardening implementation status](./docs/HARDENING_IMPLEMENTATION_STATUS_2026-04-25.md) -- [Swarm critique report](./docs/SWARM_CRITIQUE_REPORT_2026-04-24.md) +The docs and UI should avoid stronger claims such as guaranteed meetings, fully autonomous outreach, or outcome-based billing unless the implementation is explicitly verified. ## Local Development -Install dependencies from the repository root: +Install dependencies from the repo root: ```bash npm install @@ -89,102 +64,74 @@ Start the local beta stack: npm run beta:start ``` -This starts or reuses Postgres, Redis, `apps/web`, and `apps/api`, then pushes the API Prisma schema to the local database. +This brings up or reuses: -Start web, API, and optional edge runtime: +- Postgres +- Redis +- `apps/web` +- `apps/api` + +To include the optional edge runtime: ```bash npm run beta:start:all ``` -## Build And Verification +## Verification Commands + +Useful local checks: ```bash -npm run build:web -npm run build:api -npm run typecheck:web -npm run typecheck:api +npm run readiness:audit --workspace apps/api +npm run lint --workspace apps/web +npm run test:coverage --workspace apps/web +npm run test:e2e --workspace apps/web -- e2e/auth.spec.ts e2e/dashboard.spec.ts ``` -App-level checks are available from each workspace: +## Readiness Snapshot -```bash -cd apps/web -npm run test:unit -npm run test:e2e +The repo is not judged by the API readiness audit alone. -cd ../api -npm run typecheck -``` +### Green -## Repository Structure - -```text -fullstack/ -|-- apps/ -| |-- web/ # Next.js web app -| |-- api/ # Fastify API service -| |-- edge-fastapi/ # Optional private edge runtime -| -|-- packages/ # Shared packages and contracts -|-- docs/ # Documentation, diagrams, runbooks -|-- scripts/ # Repository orchestration scripts -|-- docker/ # Docker support files -|-- .github/ # GitHub Actions workflows -| -|-- docker-compose.yml # Local infrastructure and services -|-- MASTER_SYSTEM_ARCHITECTURE.md -|-- README.md -|-- package.json # Workspace scripts -|-- package-lock.json -``` +- required local runtime stack boots with Docker-backed Postgres and Redis +- API readiness audit passes at `100/100` +- architecture is cleanly split by service boundary +- edge runtime is optional and can remain private +- AI guardrails, team scoping, and landing sanitization are present in the active code path -See [docs/SIMPLE_REPO_TREE.md](./docs/SIMPLE_REPO_TREE.md) for the expanded service map. +### Still Blocking Broader Launch + +- web coverage is currently flaky in this tree: `health-route`, `metrics-route`, and `worker-dispatch` tests timed out during reassessment +- no newly confirmed green GitHub Actions run for `CI`, `Playwright`, and `docker-ghcr` after the latest local changes +- dependency security debt remains, especially in the API dependency graph +- some older docs previously described a stale managed-runtime/control-plane architecture and have now been corrected ## Deployment Model -Deploy each app as a separate service from this single repository: +Deploy each app separately from this monorepo: -| Service | Root directory | Visibility | +| Service | Root | Visibility | | --- | --- | --- | | Web | `apps/web` | Public | | API | `apps/api` | Public | -| Edge | `apps/edge-fastapi` | Private/internal, optional | -| Postgres | Managed database | Private | -| Redis | Managed cache/queue | Private, optional | - -Use path-based deploy triggers: - -- `apps/web/**` deploys web. -- `apps/api/**` deploys API. -- `apps/edge-fastapi/**` deploys edge. -- Shared changes such as `package-lock.json`, root scripts, or Prisma schema changes deploy affected services. - -## Docker And Registry - -- Local image builds are available via `npm run docker:web`, `npm run docker:api`, and `npm run docker:edge`. -- GitHub Container Registry publishing is defined in `.github/workflows/docker-ghcr.yml`. -- The web image builds from the monorepo root context so it can use the workspace lockfile consistently. -- On pushes to `main`, the workflow builds and publishes `web`, `api`, and `edge-fastapi` images to `ghcr.io/craftmyfunnelai-outreach/fullstack/*`. - -## Documentation Map - -- [Master system architecture](./MASTER_SYSTEM_ARCHITECTURE.md) -- [Mermaid architecture diagram](./docs/architecture-diagram.md) -- [AI guardrails and token usage](./docs/AI_GUARDRAILS_AND_TOKEN_USAGE.md) -- [Hardening implementation status](./docs/HARDENING_IMPLEMENTATION_STATUS_2026-04-25.md) -- [Swarm critique report](./docs/SWARM_CRITIQUE_REPORT_2026-04-24.md) -- [Netjana signal integration plan](./docs/NETJANA_SIGNAL_INTEGRATION_PLAN.md) -- [Landing Agent architecture](./docs/landing-agent-architecture.md) -- [Landing Agent API examples](./docs/landing-agent-api-examples.md) -- [Repository structure](./docs/SIMPLE_REPO_TREE.md) -- [Deployment runbook](./docs/DEPLOYMENT_RUNBOOK.md) -- [CI verification](./docs/CI_VERIFICATION.md) -- [Setup guide](./docs/SETUP.md) - -## Infrastructure Expectations - -- Postgres is required for real runtime functionality. -- Redis is optional for cache and queue features; the app should boot without Redis unless a specific workflow provisions it. -- CI jobs that need Postgres or Redis should define GitHub Actions `services:` containers and run `prisma db push` before integration tests. -- Edge runtime is optional for the email-first beta and should remain private unless explicitly exposed. +| Edge | `apps/edge-fastapi` | Private/internal | +| Postgres | Managed service | Private | +| Redis | Managed service | Private, optional | + +Path-based deploy triggers should remain the norm: + +- `apps/web/**` -> web deploy +- `apps/api/**` -> API deploy +- `apps/edge-fastapi/**` -> edge deploy + +Shared root changes such as lockfile, root scripts, or schema changes should trigger deploys for impacted services. + +## Source Of Truth Docs + +- [MASTER_SYSTEM_ARCHITECTURE.md](./MASTER_SYSTEM_ARCHITECTURE.md) +- [docs/ARCHITECTURE.md](./docs/ARCHITECTURE.md) +- [docs/README.md](./docs/README.md) +- [docs/PRODUCTION_READINESS_ASSESSMENT_2026-06-02.md](./docs/PRODUCTION_READINESS_ASSESSMENT_2026-06-02.md) +- [docs/context/ARCHITECTURE.md](./docs/context/ARCHITECTURE.md) +- [docs/context/LAUNCH_READINESS.md](./docs/context/LAUNCH_READINESS.md) diff --git a/apps/README.md b/apps/README.md index 9686fce7..43b8d4a8 100644 --- a/apps/README.md +++ b/apps/README.md @@ -1,27 +1,37 @@ # Apps Folder -This directory contains the three deployable applications in this monorepo: +This directory contains the deployable services in the CraftMyFunnel monorepo. -- `web` - Next.js app (public UI) -- `api` - Fastify app (public API) -- `edge-fastapi` - FastAPI app (private optional edge runtime) +| App | Path | Role | Required | +| --- | --- | --- | --- | +| Web | `apps/web` | Public Next.js UI, dashboard, setup, public landing pages | Yes | +| API | `apps/api` | Public Fastify API, workers, Prisma, webhooks, readiness checks | Yes | +| Edge | `apps/edge-fastapi` | Optional private FastAPI edge runtime | No | -## Independent deploy model +## Deployment Model -Each app is deployed separately, even though they live in one git repository. +Each app deploys independently even though the code lives in one repository. -- web service uses `apps/web` as its deploy root -- api service uses `apps/api` as its deploy root -- edge service uses `apps/edge-fastapi` as its deploy root +- web deploy root: `apps/web` +- API deploy root: `apps/api` +- edge deploy root: `apps/edge-fastapi` -## Why keep them in one repo +## Local Runtime -- one source of truth for contracts and schemas -- one PR can change all required layers safely -- CI can still deploy services independently using path filters +From the repo root: -## Local split runtime +```bash +npm run beta:start +``` + +To include the optional edge runtime: ```bash -docker compose -f docker-compose.split.yml up -d +npm run beta:start:all ``` + +## Notes + +- Postgres is required for real product behavior. +- Redis is recommended but optional. +- Edge should remain private unless there is a clear product or operational need to expose it. diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 966f4cb0..3d8b113d 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.7 # Dependencies stage (workspace-aware install for monorepo) FROM node:22-alpine AS deps WORKDIR /repo @@ -7,7 +8,12 @@ COPY apps/api/package.json ./apps/api/package.json COPY apps/web/package.json ./apps/web/package.json COPY packages ./packages -RUN npm ci --workspace apps/api --include-workspace-root --no-audit --no-fund --legacy-peer-deps +RUN sh -c 'for attempt in 1 2 3; do \ + npm ci --workspace apps/api --include-workspace-root --no-audit --no-fund --legacy-peer-deps --prefer-offline --fetch-retries=5 --fetch-retry-maxtimeout=120000 && exit 0; \ + echo "npm ci failed on attempt ${attempt}, retrying..." >&2; \ + sleep 5; \ + done; \ + exit 1' # Build artifacts needed at runtime FROM node:22-alpine AS builder diff --git a/apps/api/package.json b/apps/api/package.json index 45396b51..e5f80a4f 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -3,29 +3,24 @@ "private": true, "type": "module", "scripts": { - "dev": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PORT=3001 -- node ../../node_modules/tsx/dist/cli.mjs --watch server.ts", - "start": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PORT=3001 -- node ../../node_modules/tsx/dist/cli.mjs server.ts", + "dev": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PORT=3001 -- tsx --watch server.ts", + "start": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PORT=3001 -- tsx server.ts", "build": "tsc -p tsconfig.strict.json", "typecheck": "tsc -p tsconfig.strict.json --noEmit", - "readiness:audit": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- node -r dotenv/config ../../node_modules/tsx/dist/cli.mjs src/scripts/validate-production-readiness.ts", - "test": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs run --config vitest.config.ts", - "test:routes": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs run --config vitest.config.ts" + "readiness:audit": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- tsx -r dotenv/config src/scripts/validate-production-readiness.ts", + "test": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest run --config vitest.config.ts", + "test:routes": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest run --config vitest.config.ts" }, "dependencies": { "@anthropic-ai/sdk": "^0.71.2", "@fastify/cors": "^10.0.0", "@fastify/helmet": "^12.0.0", "@fastify/rate-limit": "^10.0.0", - "@genkit-ai/ai": "^1.28.0", - "@genkit-ai/core": "^1.28.0", - "@genkit-ai/dotprompt": "^0.9.12", - "@genkit-ai/flow": "^0.5.17", - "@genkit-ai/googleai": "^1.28.0", "@google/generative-ai": "^0.24.1", "@hubspot/api-client": "^13.4.0", "@next-auth/prisma-adapter": "^1.0.7", "@prisma/adapter-pg": "^7.5.0", - "@prisma/client": "^7.5.0", + "@prisma/client": "^7.8.0", "axios": "^1.13.2", "bcryptjs": "^3.0.3", "cron-parser": "^5.4.0", @@ -33,19 +28,19 @@ "dotenv": "^17.0.0", "eventsource": "^4.1.0", "fastify": "^5.1.0", - "genkit": "^1.28.0", - "geoip-lite": "^1.4.10", + "geoip-lite": "^2.0.2", "googleapis": "^171.4.0", + "grapesjs": "^0.23.2", "groq-sdk": "^0.37.0", "ioredis": "^5.9.1", "lru-cache": "^11.2.4", - "next": "^16.2.2", - "next-auth": "^4.24.13", - "nodemailer": "^7.0.13", + "next": "^16.2.7", + "next-auth": "^4.24.14", + "nodemailer": "^8.0.10", "openai": "^6.16.0", "papaparse": "^5.5.3", "pg": "^8.20.0", - "prisma": "^7.5.0", + "prisma": "^7.8.0", "prom-client": "^15.1.3", "puppeteer": "^23.11.1", "puppeteer-extra": "^3.3.6", @@ -58,7 +53,6 @@ "tsx": "^4.21.0", "uuid": "^13.0.0", "winston": "^3.19.0", - "xlsx": "^0.18.5", "zod": "^4.3.6" }, "devDependencies": { @@ -68,6 +62,6 @@ "@types/pg": "^8.15.6", "pino-pretty": "^13.0.0", "typescript": "5.9.3", - "vitest": "^4.0.16" + "vitest": "^4.1.8" } } diff --git a/apps/api/routes/metrics/route.test.ts b/apps/api/routes/metrics/route.test.ts new file mode 100644 index 00000000..93143378 --- /dev/null +++ b/apps/api/routes/metrics/route.test.ts @@ -0,0 +1,37 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/metrics", () => ({ + getMetrics: vi.fn().mockResolvedValue("# HELP api_metric\n# TYPE api_metric counter\n"), +})); + +function request(headers?: HeadersInit) { + return new Request("http://localhost:3001/api/metrics", headers ? { headers } : undefined); +} + +describe("/api/metrics", () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + vi.resetModules(); + process.env = { ...originalEnv, NODE_ENV: "production", METRICS_TOKEN: "test-token" }; + }); + + it("rejects requests without the metrics bearer token", async () => { + const { GET } = await import("./route"); + + const response = await GET(request()); + + expect(response.status).toBe(401); + }); + + it("returns prometheus metrics to authorized scrapers", async () => { + const { GET } = await import("./route"); + + const response = await GET(request({ Authorization: "Bearer test-token" })); + const body = await response.text(); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/plain"); + expect(body).toContain("# HELP"); + }); +}); diff --git a/apps/api/routes/metrics/route.ts b/apps/api/routes/metrics/route.ts index 17143f9c..17b45c25 100644 --- a/apps/api/routes/metrics/route.ts +++ b/apps/api/routes/metrics/route.ts @@ -3,7 +3,22 @@ import { getMetrics } from '@/lib/metrics'; export const dynamic = 'force-dynamic'; -export async function GET() { +function isAuthorized(request: Request) { + const token = process.env['METRICS_TOKEN']; + const authorization = request.headers.get('authorization'); + + if (!token) { + return process.env['NODE_ENV'] !== 'production'; + } + + return authorization === `Bearer ${token}`; +} + +export async function GET(request: Request) { + if (!isAuthorized(request)) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + try { const metrics = await getMetrics(); return new NextResponse(metrics, { diff --git a/apps/api/server.ts b/apps/api/server.ts index 4295aea9..07d2914a 100644 --- a/apps/api/server.ts +++ b/apps/api/server.ts @@ -154,6 +154,8 @@ const nextAdapter = (handler: any, registeredPath: string) => async (request: an "/webhooks", "/auth", "/register", + "/health", + "/metrics", // "/test-auth" removed for production security "/scheduler", "/integrations/google/oauth/callback", diff --git a/apps/api/src/lib/db.ts b/apps/api/src/lib/db.ts index 935fbde0..6e8fbd3e 100644 --- a/apps/api/src/lib/db.ts +++ b/apps/api/src/lib/db.ts @@ -14,10 +14,18 @@ const createPrismaClient = () => { throw new Error("DATABASE_URL is not set."); } + const resolvedDatabaseUrl = + process.env["NODE_ENV"] === "production" + ? databaseUrl + : databaseUrl.replace("@localhost:", "@127.0.0.1:"); + const { Pool } = require("pg"); const { PrismaPg } = require("@prisma/adapter-pg"); const pool = new Pool({ - connectionString: databaseUrl, + connectionString: resolvedDatabaseUrl, + max: 10, + idleTimeoutMillis: 30_000, + connectionTimeoutMillis: 5_000, }); const adapter = new PrismaPg(pool); diff --git a/apps/api/src/lib/redis.ts b/apps/api/src/lib/redis.ts index e89d439f..98b03c75 100644 --- a/apps/api/src/lib/redis.ts +++ b/apps/api/src/lib/redis.ts @@ -2,7 +2,11 @@ let redisClient: any = null; function resolveRedisUrl(): string | null { const explicit = (process.env["REDIS_URL"] || "").trim(); - if (explicit) return explicit; + if (explicit) { + return process.env["NODE_ENV"] === "production" + ? explicit + : explicit.replace("redis://localhost", "redis://127.0.0.1"); + } const inCi = process.env["CI"] === "true" || process.env["GITHUB_ACTIONS"] === "true"; const isProd = process.env["NODE_ENV"] === "production"; @@ -15,7 +19,7 @@ function resolveRedisUrl(): string | null { return null; } - return "redis://localhost:6379"; + return "redis://127.0.0.1:6379"; } export async function getRedisClient() { @@ -65,6 +69,10 @@ export async function getRedisClient() { if (process.env["NODE_ENV"] !== "test") { console.error("Failed to connect to Redis:", error); } + if (redisClient) { + redisClient.destroy(); + redisClient = null; + } // Do not throw, allow app to start without Redis } diff --git a/apps/docker-compose.split.yml b/apps/docker-compose.split.yml index 6f5e9027..a4f9552e 100644 --- a/apps/docker-compose.split.yml +++ b/apps/docker-compose.split.yml @@ -42,7 +42,12 @@ services: # Edge FastAPI — Sovereign AI / PII Firewall # ───────────────────────────────────────────── edge-fastapi: - build: ./edge-fastapi + build: + context: ./edge-fastapi + args: + REQUIREMENTS_FILE: ${EDGE_REQUIREMENTS_FILE:-requirements.runtime.txt} + FORCE_CMAKE: ${EDGE_FORCE_CMAKE:-0} + CMAKE_ARGS: "${EDGE_CMAKE_ARGS:--DGGML_NATIVE=OFF -DGGML_OPENMP=ON}" image: craftmyfunnel-edge-fastapi:split container_name: craftmyfunnel-edge-fastapi-split restart: unless-stopped @@ -51,7 +56,16 @@ services: EDGE_MODE: ${EDGE_MODE:-false} HARDWARE_SIGNATURE: uuid-550e8400-e29b-41d4-a716-446655440000 OFFLINE_MODEL_PATH: /app/models/custom/Phi-3-Custom.gguf - EDGE_API_KEY: ${EDGE_API_KEY:?EDGE_API_KEY is required} + EDGE_API_KEY: ${EDGE_API_KEY:-} + EDGE_REQUIRE_API_KEY: ${EDGE_REQUIRE_API_KEY:-true} + EDGE_VAULT_KEY: ${EDGE_VAULT_KEY:-} + EDGE_PRELOAD_MODELS: ${EDGE_PRELOAD_MODELS:-none} + EDGE_MAX_TEXT_CHARS: ${EDGE_MAX_TEXT_CHARS:-12000} + LLM_THREADS: ${LLM_THREADS:-4} + LLM_CONTEXT_TOKENS: ${LLM_CONTEXT_TOKENS:-1536} + LLM_BATCH_SIZE: ${LLM_BATCH_SIZE:-128} + DB_POOL_SIZE: ${EDGE_DB_POOL_SIZE:-2} + DB_MAX_OVERFLOW: ${EDGE_DB_MAX_OVERFLOW:-2} EDGE_EXECUTE_ENABLED: ${EDGE_EXECUTE_ENABLED:-false} volumes: - ../training/models:/app/models/custom @@ -73,7 +87,9 @@ services: # API — Fastify + Prisma # ───────────────────────────────────────────── api: - build: ./api + build: + context: .. + dockerfile: apps/api/Dockerfile image: craftmyfunnel-api:split container_name: craftmyfunnel-api-split restart: unless-stopped @@ -108,7 +124,11 @@ services: # Web — Next.js Frontend # ───────────────────────────────────────────── web: - build: ./web + build: + context: .. + dockerfile: apps/web/Dockerfile + args: + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:3000/api/proxy} image: craftmyfunnel-web:split container_name: craftmyfunnel-web-split restart: unless-stopped @@ -116,10 +136,12 @@ services: NODE_ENV: production NEXT_TELEMETRY_DISABLED: "1" DATABASE_URL: postgresql://postgres:postgres@postgres:5432/craftmyfunnel - NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://api:3001} + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:3000/api/proxy} + API_INTERNAL_ORIGIN: http://api:3001 EDGE_NODE_URI: http://edge-fastapi:8000 NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000} NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:?NEXTAUTH_SECRET is required} + ENCRYPTION_KEY: ${ENCRYPTION_KEY:?ENCRYPTION_KEY is required} ports: - "3000:3000" depends_on: diff --git a/apps/edge-fastapi/.env.example b/apps/edge-fastapi/.env.example index 906c6c03..d898c064 100644 --- a/apps/edge-fastapi/.env.example +++ b/apps/edge-fastapi/.env.example @@ -1,3 +1,16 @@ DATABASE_URL=postgresql://user:password@host:5432/craftmyfunnel_api?sslmode=require EDGE_MODE=disabled +EDGE_REQUIRE_API_KEY=true +EDGE_API_KEY=change-me-long-random-token +EDGE_REQUIRE_VAULT_KEY=true +EDGE_VAULT_KEY=generate-with-python-fernet HARDWARE_SIGNATURE=uuid-550e8400-e29b-41d4-a716-446655440000 +EDGE_PRELOAD_MODELS=none +EDGE_MAX_TEXT_CHARS=12000 +EDGE_SPACY_MODEL=en_core_web_sm +OFFLINE_MODEL_PATH=/app/models/Phi-3-mini-4k-instruct-q4.gguf +LLM_THREADS=4 +LLM_CONTEXT_TOKENS=1536 +LLM_BATCH_SIZE=128 +DB_POOL_SIZE=2 +DB_MAX_OVERFLOW=2 diff --git a/apps/edge-fastapi/Dockerfile b/apps/edge-fastapi/Dockerfile index e44e3eb0..9b7280ec 100644 --- a/apps/edge-fastapi/Dockerfile +++ b/apps/edge-fastapi/Dockerfile @@ -6,6 +6,11 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 +ARG CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_OPENMP=ON" +ARG FORCE_CMAKE=0 +ENV CMAKE_ARGS=${CMAKE_ARGS} \ + FORCE_CMAKE=${FORCE_CMAKE} + WORKDIR /app # Install build-only system dependencies @@ -14,6 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ g++ \ curl \ + libopenblas-dev \ && rm -rf /var/lib/apt/lists/* # Pre-install specific libraries to avoid huge layer rebuilds @@ -31,7 +37,10 @@ FROM python:3.11-slim AS runner ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ HOME=/home/appuser \ - PATH="/home/appuser/.local/bin:${PATH}" + PATH="/home/appuser/.local/bin:${PATH}" \ + OMP_NUM_THREADS=4 \ + OPENBLAS_NUM_THREADS=4 \ + TOKENIZERS_PARALLELISM=false WORKDIR /app @@ -42,6 +51,7 @@ RUN adduser --system --group --home /home/appuser appuser && \ apt-get update && apt-get install -y --no-install-recommends \ libssl3 \ libgomp1 \ + libopenblas0-pthread \ libpq-dev \ curl \ && rm -rf /var/lib/apt/lists/* diff --git a/apps/edge-fastapi/README.md b/apps/edge-fastapi/README.md index 9456e100..8c391322 100644 --- a/apps/edge-fastapi/README.md +++ b/apps/edge-fastapi/README.md @@ -8,15 +8,47 @@ Standalone edge runtime extracted from `services/edge-node`. python -m venv .venv source .venv/bin/activate pip install -r requirements.txt +python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" uvicorn main:app --host 0.0.0.0 --port 8000 ``` +For `EDGE_MODE=enabled`, set `EDGE_API_KEY` and `EDGE_VAULT_KEY`. Sensitive routes accept either +`X-API-Key: ` or `Authorization: Bearer `. + ## Build Docker image ```bash docker build -t craftmyfunnel-edge-fastapi:split . ``` +## Raspberry Pi 5 build + +Use the Pi profile for ARM64 edge mode. It keeps the image lighter by installing PII masking and +`llama-cpp-python` for GGUF micro LLM inference, while leaving the heavier sentence-transformer critic +as an optional server-class feature. + +```bash +docker buildx build \ + --platform linux/arm64 \ + --build-arg REQUIREMENTS_FILE=requirements.pi.txt \ + --build-arg FORCE_CMAKE=1 \ + --build-arg CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_OPENMP=ON" \ + -t craftmyfunnel-edge-fastapi:pi5 . +``` + +Recommended Pi 5 env: + +```bash +EDGE_MODE=enabled +EDGE_PRELOAD_MODELS=none +EDGE_SPACY_MODEL=en_core_web_sm +LLM_THREADS=4 +LLM_CONTEXT_TOKENS=1536 +LLM_BATCH_SIZE=128 +DB_POOL_SIZE=2 +DB_MAX_OVERFLOW=2 +``` + ## Default port - `8000` diff --git a/apps/edge-fastapi/database.py b/apps/edge-fastapi/database.py index 29de9447..432df944 100644 --- a/apps/edge-fastapi/database.py +++ b/apps/edge-fastapi/database.py @@ -1,5 +1,5 @@ import os -from sqlalchemy import create_engine, Column, String, Float, Integer, JSON, DateTime +from sqlalchemy import create_engine, Column, String, Float, Integer, JSON, DateTime, Text from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from pgvector.sqlalchemy import Vector @@ -12,10 +12,10 @@ engine = create_engine( DATABASE_URL, - pool_size=3, - max_overflow=5, + pool_size=int(os.getenv("DB_POOL_SIZE", "2")), + max_overflow=int(os.getenv("DB_MAX_OVERFLOW", "2")), pool_pre_ping=True, - pool_recycle=300, + pool_recycle=int(os.getenv("DB_POOL_RECYCLE_SECONDS", "300")), ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) Base = declarative_base() @@ -35,12 +35,15 @@ class PIITokenMap(Base): __tablename__ = "pii_token_map" token_id = Column(String, primary_key=True) - original_text = Column(String) + original_text = Column(Text) token_type = Column(String) # PERSON, EMAIL, etc. session_id = Column(String, index=True) created_at = Column(DateTime, default=datetime.datetime.utcnow) def init_db(): + if engine.dialect.name == "postgresql": + with engine.begin() as connection: + connection.exec_driver_sql("CREATE EXTENSION IF NOT EXISTS vector") Base.metadata.create_all(bind=engine) def get_db(): diff --git a/apps/edge-fastapi/main.py b/apps/edge-fastapi/main.py index 5b909d7c..3e9596c3 100644 --- a/apps/edge-fastapi/main.py +++ b/apps/edge-fastapi/main.py @@ -2,12 +2,13 @@ import uuid import logging import time +import hmac from typing import Dict, List, Optional from enum import Enum from fastapi import FastAPI, HTTPException, Depends, Header from sqlalchemy.orm import Session from sqlalchemy import text -from pydantic import BaseModel +from pydantic import BaseModel, Field from database import init_db, get_db, SessionLocal # Logging @@ -30,6 +31,9 @@ async def dispatch(self, request, call_next): EDGE_MODE = os.getenv("EDGE_MODE", "disabled").lower() in {"1", "true", "edge", "enabled"} EDGE_EXECUTE_ENABLED = os.getenv("EDGE_EXECUTE_ENABLED", "false").lower() == "true" EDGE_API_KEY = os.getenv("EDGE_API_KEY") +EDGE_REQUIRE_API_KEY = os.getenv("EDGE_REQUIRE_API_KEY", "true").lower() == "true" +EDGE_REQUIRE_VAULT_KEY = os.getenv("EDGE_REQUIRE_VAULT_KEY", "true").lower() == "true" +MAX_TEXT_CHARS = int(os.getenv("EDGE_MAX_TEXT_CHARS", "12000")) # [MED-5] Startup assertion: actuator must have a key if enabled if EDGE_EXECUTE_ENABLED and not EDGE_API_KEY: @@ -38,6 +42,18 @@ async def dispatch(self, request, call_next): "The browser actuator cannot run unauthenticated. Set EDGE_API_KEY or disable EDGE_EXECUTE_ENABLED." ) +if EDGE_MODE and EDGE_REQUIRE_API_KEY and not EDGE_API_KEY: + raise RuntimeError( + "SECURITY ERROR: EDGE_MODE is enabled but EDGE_API_KEY is not set. " + "Set EDGE_API_KEY or explicitly set EDGE_REQUIRE_API_KEY=false for a local-only lab run." + ) + +if EDGE_MODE and EDGE_REQUIRE_VAULT_KEY and not os.getenv("EDGE_VAULT_KEY"): + raise RuntimeError( + "SECURITY ERROR: EDGE_MODE is enabled but EDGE_VAULT_KEY is not set. " + "Generate one with: python -c \"from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())\"" + ) + app = FastAPI(title="CraftMyFunnel Edge Node") app.add_middleware(CorrelationMiddleware) @@ -67,7 +83,7 @@ def filter(self, record): # --- Schemas --- class SanitizeRequest(BaseModel): - text: str + text: str = Field(min_length=1) session_id: Optional[str] = None class SanitizeResponse(BaseModel): @@ -76,7 +92,7 @@ class SanitizeResponse(BaseModel): stats: Dict[str, int] class CritiqueRequest(BaseModel): - text: str + text: str = Field(min_length=1) context: Optional[str] = None class CritiqueResponse(BaseModel): @@ -90,7 +106,7 @@ class StatusResponse(BaseModel): services: List[str] class OfflineRequest(BaseModel): - prompt: str + prompt: str = Field(min_length=1) class OfflineResponse(BaseModel): text: str @@ -108,6 +124,31 @@ class ExecuteRequest(BaseModel): # --- Endpoints --- +def require_edge_api_key( + x_api_key: Optional[str] = Header(None), + authorization: Optional[str] = Header(None), +): + if not EDGE_API_KEY: + if EDGE_REQUIRE_API_KEY and EDGE_MODE: + raise HTTPException(status_code=503, detail="EDGE_API_KEY is not configured") + return + + bearer = None + if authorization and authorization.lower().startswith("bearer "): + bearer = authorization[7:] + + presented = x_api_key or bearer + if not presented or not hmac.compare_digest(presented, EDGE_API_KEY): + raise HTTPException(status_code=401, detail="Invalid API key") + + +def enforce_text_limit(value: str): + if len(value) > MAX_TEXT_CHARS: + raise HTTPException( + status_code=413, + detail=f"Text exceeds EDGE_MAX_TEXT_CHARS ({MAX_TEXT_CHARS})", + ) + def init_db_with_retry(retries=5, delay=3): for attempt in range(retries): try: @@ -124,7 +165,7 @@ def startup_event(): logger.info("Initializing Edge Database...") init_db_with_retry() if EDGE_MODE and local_intelligence: - local_intelligence.load_models() + local_intelligence.warmup(os.getenv("EDGE_PRELOAD_MODELS", "none").lower()) logger.info("Edge mode enabled.") else: logger.info("Edge mode disabled.") @@ -161,11 +202,12 @@ def capabilities(): } } -@app.post("/v1/sanitize", response_model=SanitizeResponse) +@app.post("/v1/sanitize", response_model=SanitizeResponse, dependencies=[Depends(require_edge_api_key)]) def sanitize_text(request: SanitizeRequest): """Deep Tech: Semantic Masking with Metadata Injection""" if not EDGE_MODE or not local_intelligence: raise HTTPException(status_code=503, detail="Edge features disabled") + enforce_text_limit(request.text) session_id = request.session_id or str(uuid.uuid4()) sanitized_text, stats = local_intelligence.sanitize_and_tag(request.text, session_id) @@ -175,11 +217,12 @@ def sanitize_text(request: SanitizeRequest): "stats": stats } -@app.post("/v1/critique", response_model=CritiqueResponse) +@app.post("/v1/critique", response_model=CritiqueResponse, dependencies=[Depends(require_edge_api_key)]) def critique_response(request: CritiqueRequest): """Deep Tech: Adversarial Judge""" if not EDGE_MODE or not local_intelligence: raise HTTPException(status_code=503, detail="Edge features disabled") + enforce_text_limit(request.text) result = local_intelligence.critique_response(request.text) return { "status": result["status"], @@ -187,11 +230,12 @@ def critique_response(request: CritiqueRequest): "reason": result.get("reason") } -@app.post("/v1/generate_offline", response_model=OfflineResponse) +@app.post("/v1/generate_offline", response_model=OfflineResponse, dependencies=[Depends(require_edge_api_key)]) def generate_offline(request: OfflineRequest): """Deep Tech: Offline Fallback (Quantized SLM)""" if not EDGE_MODE or not local_intelligence: raise HTTPException(status_code=503, detail="Edge features disabled") + enforce_text_limit(request.prompt) output = local_intelligence.generate_offline(request.prompt) return {"text": output} @@ -199,7 +243,7 @@ class ReidentifyRequest(BaseModel): token: str session_id: Optional[str] = None -@app.post("/v1/reidentify") +@app.post("/v1/reidentify", dependencies=[Depends(require_edge_api_key)]) def reidentify_token(request: ReidentifyRequest): """Deep Tech: Protocol Break - Identity Vault Lookup""" if not EDGE_MODE or not local_intelligence: @@ -210,16 +254,17 @@ def reidentify_token(request: ReidentifyRequest): return {"original": original} class ScoreRequest(BaseModel): - text: str + text: str = Field(min_length=1) class ScoreResponse(BaseModel): score: int -@app.post("/v1/score_intent", response_model=ScoreResponse) +@app.post("/v1/score_intent", response_model=ScoreResponse, dependencies=[Depends(require_edge_api_key)]) def score_intent(request: ScoreRequest): """Deep Tech: Karmic Friction Analysis (Offline SLM)""" if not EDGE_MODE or not local_intelligence: raise HTTPException(status_code=503, detail="Edge features disabled") + enforce_text_limit(request.text) score = local_intelligence.score_intent(request.text) return {"score": score} @@ -227,19 +272,19 @@ def score_intent(request: ScoreRequest): class SearchRequest(BaseModel): query: str - limit: int = 5 + limit: int = Field(default=5, ge=1, le=25) class SearchResponse(BaseModel): results: List[Dict] -@app.post("/search", response_model=SearchResponse) +@app.post("/search", response_model=SearchResponse, dependencies=[Depends(require_edge_api_key)]) def search_golden_records(request: SearchRequest, db: Session = Depends(get_db)): """Semantic Search against Local Golden Records""" if not EDGE_MODE or not local_intelligence: raise HTTPException(status_code=503, detail="Edge features disabled") try: if not local_intelligence.embedding_model: - local_intelligence.load_models() + local_intelligence.load_embedding_model() vector = local_intelligence.embedding_model.encode(request.query).tolist() vector_str = str(vector) @@ -261,15 +306,12 @@ def search_golden_records(request: SearchRequest, db: Session = Depends(get_db)) logger.error(f"Search failed: {e}") return {"results": []} -@app.post("/execute") -def execute_browser_action(request: ExecuteRequest, x_api_key: str = Header(None)): +@app.post("/execute", dependencies=[Depends(require_edge_api_key)]) +def execute_browser_action(request: ExecuteRequest): """Secured Proxy for Browser Node (Puppeteer)""" if not EDGE_EXECUTE_ENABLED: raise HTTPException(status_code=403, detail="Edge execution is disabled") - if EDGE_API_KEY and x_api_key != EDGE_API_KEY: - raise HTTPException(status_code=401, detail="Invalid API Key") - logger.info(f"PHYSICAL ACTUATOR: Executing {request.action} with {request.payload}") return { diff --git a/apps/edge-fastapi/requirements.pi.txt b/apps/edge-fastapi/requirements.pi.txt new file mode 100644 index 00000000..86842bb5 --- /dev/null +++ b/apps/edge-fastapi/requirements.pi.txt @@ -0,0 +1,12 @@ +fastapi +uvicorn +pydantic +cryptography +sqlalchemy +psycopg2-binary +pgvector +presidio-analyzer +presidio-anonymizer +spacy==3.7.0 +https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.0/en_core_web_sm-3.7.0.tar.gz +llama-cpp-python diff --git a/apps/edge-fastapi/requirements.txt b/apps/edge-fastapi/requirements.txt index 241f160d..3e517a9a 100644 --- a/apps/edge-fastapi/requirements.txt +++ b/apps/edge-fastapi/requirements.txt @@ -1,6 +1,7 @@ fastapi uvicorn pydantic +cryptography presidio-analyzer presidio-anonymizer spacy==3.7.0 diff --git a/apps/edge-fastapi/services/local_intelligence.py b/apps/edge-fastapi/services/local_intelligence.py index b519c69d..acd1162c 100644 --- a/apps/edge-fastapi/services/local_intelligence.py +++ b/apps/edge-fastapi/services/local_intelligence.py @@ -1,108 +1,177 @@ -import os import logging -import random -from typing import List, Dict, Tuple, Optional -from datetime import datetime -import uuid +import os +from typing import Dict, Optional, Tuple -# ML / AI Libraries +from cryptography.fernet import Fernet, InvalidToken from presidio_analyzer import AnalyzerEngine -from sentence_transformers import SentenceTransformer - -# Database -from sqlalchemy.orm import Session from sqlalchemy import text as sql_text + from database import GoldenRecord, PIITokenMap logger = logging.getLogger(__name__) +DEFAULT_ENTITY_TYPES = [ + "PERSON", + "EMAIL_ADDRESS", + "PHONE_NUMBER", + "LOCATION", + "URL", + "IP_ADDRESS", + "CREDIT_CARD", + "CRYPTO", + "IBAN_CODE", + "US_SSN", + "IN_PAN", + "IN_AADHAAR", +] + +ENCRYPTED_PREFIX = "enc:v1:" + + class LocalIntelligenceService: def __init__(self, db_session_factory): self.db_session_factory = db_session_factory - self.analyzer = AnalyzerEngine() + self.analyzer = None self.embedding_model = None self.llm = None - - # Lazy Loading configurations - self._models_loaded = False + self.fernet = self._build_fernet() + self._analyzer_loaded = False + self._embedding_loaded = False + self._llm_loaded = False - def load_models(self): - """Lazy load heavy models to save startup time/RAM""" - if self._models_loaded: - return + def _build_fernet(self) -> Optional[Fernet]: + key = os.getenv("EDGE_VAULT_KEY") + if not key: + return None + try: + return Fernet(key.encode("utf-8")) + except Exception as exc: + raise RuntimeError("EDGE_VAULT_KEY must be a valid Fernet key") from exc + + def _encrypt_pii(self, value: str) -> str: + if not self.fernet: + raise RuntimeError("EDGE_VAULT_KEY is required before storing PII tokens") + encrypted = self.fernet.encrypt(value.encode("utf-8")).decode("utf-8") + return f"{ENCRYPTED_PREFIX}{encrypted}" - logger.info("Initializing Local Intelligence Models...") - - # 1. Critic Model (SentenceTransformers) + def _decrypt_pii(self, value: str) -> Optional[str]: + if not value.startswith(ENCRYPTED_PREFIX): + return value + if not self.fernet: + logger.warning("PII token cannot be decrypted because EDGE_VAULT_KEY is not configured") + return None try: - self.embedding_model = SentenceTransformer('all-MiniLM-L6-v2') - logger.info("✅ Critic Model Loaded (all-MiniLM-L6-v2)") - except Exception as e: - logger.error(f"❌ Failed to load Critic Model: {e}") + encrypted = value.removeprefix(ENCRYPTED_PREFIX) + return self.fernet.decrypt(encrypted.encode("utf-8")).decode("utf-8") + except InvalidToken: + logger.warning("PII token decrypt failed") + return None - # 2. Offline LLM (Phi-3-Mini GGUF) - model_path = os.getenv("OFFLINE_MODEL_PATH", "./models/Phi-3-mini-4k-instruct.gguf") - - if os.path.exists(model_path): + def load_analyzer(self): + if self._analyzer_loaded: + return + logger.info("Initializing PII analyzer") + preferred_model = os.getenv("EDGE_SPACY_MODEL") + candidate_models = [preferred_model] if preferred_model else ["en_core_web_sm", "en_core_web_lg"] + last_error = None + + for model_name in candidate_models: try: - from llama_cpp import Llama - self.llm = Llama( - model_path=model_path, - n_ctx=2048, - n_threads=4, - n_gpu_layers=0 + from presidio_analyzer.nlp_engine import NlpEngineProvider + + provider = NlpEngineProvider( + nlp_configuration={ + "nlp_engine_name": "spacy", + "models": [{"lang_code": "en", "model_name": model_name}], + } ) - logger.info(f"✅ Offline LLM Loaded ({model_path})") - except Exception as e: - logger.error(f"❌ Failed to load Offline LLM: {e}") - else: - logger.warning(f"⚠️ Offline model not found at {model_path}.") - - self._models_loaded = True - - # --- Task 1: Sovereign Firewall (Taxonomy-Aware Masking) --- - def sanitize_and_tag(self, text: str, session_id: str) -> Tuple[str, Dict]: - """ - Detects PII and replaces with Metadata-Injected Tokens. - """ - self.load_models() - - results = self.analyzer.analyze(text=text, language='en') - sorted_results = sorted(results, key=lambda x: x.start, reverse=True) - + self.analyzer = AnalyzerEngine(nlp_engine=provider.create_engine()) + logger.info("PII analyzer loaded with %s", model_name) + self._analyzer_loaded = True + return + except Exception as exc: + last_error = exc + + raise RuntimeError(f"Failed to initialize PII analyzer: {last_error}") + + def load_embedding_model(self): + if self._embedding_loaded: + return + logger.info("Initializing critic embedding model") + try: + from sentence_transformers import SentenceTransformer + + self.embedding_model = SentenceTransformer("all-MiniLM-L6-v2") + logger.info("Critic model loaded") + except Exception as exc: + logger.error("Failed to load critic model: %s", exc) + self._embedding_loaded = True + + def load_llm(self): + if self._llm_loaded: + return + + model_path = os.getenv("OFFLINE_MODEL_PATH", "./models/Phi-3-mini-4k-instruct.gguf") + if not os.path.exists(model_path): + logger.warning("Offline model not found at %s", model_path) + self._llm_loaded = True + return + + try: + from llama_cpp import Llama + + self.llm = Llama( + model_path=model_path, + n_ctx=int(os.getenv("LLM_CONTEXT_TOKENS", "1536")), + n_threads=int(os.getenv("LLM_THREADS", str(min(os.cpu_count() or 4, 4)))), + n_batch=int(os.getenv("LLM_BATCH_SIZE", "128")), + n_gpu_layers=int(os.getenv("LLM_GPU_LAYERS", "0")), + verbose=os.getenv("LLM_VERBOSE", "false").lower() == "true", + ) + logger.info("Offline LLM loaded from %s", model_path) + except Exception as exc: + logger.error("Failed to load offline LLM: %s", exc) + self._llm_loaded = True + + def warmup(self, mode: str): + if mode in {"pii", "all"}: + self.load_analyzer() + if mode in {"critic", "all"}: + self.load_embedding_model() + if mode in {"llm", "all"}: + self.load_llm() + + def sanitize_and_tag(self, text: str, session_id: str) -> Tuple[str, Dict[str, int]]: + self.load_analyzer() + + entities = os.getenv("PII_ENTITY_TYPES") + entity_types = [item.strip() for item in entities.split(",")] if entities else DEFAULT_ENTITY_TYPES + results = self.analyzer.analyze(text=text, language="en", entities=entity_types) + sorted_results = sorted(results, key=lambda result: result.start, reverse=True) + sanitized_text_list = list(text) - token_stats = {} + token_stats: Dict[str, int] = {} new_db_tokens = [] - counters = {} + counters: Dict[str, int] = {} for result in sorted_results: entity_type = result.entity_type start = result.start end = result.end original_val = text[start:end] - + count = counters.get(entity_type, 0) + 1 counters[entity_type] = count - - # Metadata Injection (Role/Sector Inference) - if entity_type == "PERSON": - roles = ["Executive", "Manager", "Influencer", "Gatekeeper", "Decision Maker"] - meta_attr = f" role='{random.choice(roles)}'" - elif entity_type == "ORG": - sectors = ["Enterprise", "Mid-Market", "SMB", "Government", "Non-Profit"] - meta_attr = f" sector='{random.choice(sectors)}'" - else: - meta_attr = "" - - token_label = f"<{entity_type}_{count}{meta_attr}>" + + token_label = f"<{entity_type}_{count} confidence='{result.score:.2f}'>" sanitized_text_list[start:end] = list(token_label) - + token_id = f"{session_id}_{entity_type}_{count}" token = PIITokenMap( token_id=token_id, - original_text=original_val, + original_text=self._encrypt_pii(original_val), token_type=entity_type, - session_id=session_id + session_id=session_id, ) new_db_tokens.append(token) token_stats[entity_type] = count @@ -118,93 +187,87 @@ def sanitize_and_tag(self, text: str, session_id: str) -> Tuple[str, Dict]: return "".join(sanitized_text_list), token_stats def reidentify_token(self, token_id: str, session_id: Optional[str] = None) -> Optional[str]: - """ - Reverse lookup for a token in the local PII Vault. - """ db = self.db_session_factory() try: query = db.query(PIITokenMap).filter(PIITokenMap.token_id == token_id) if session_id: query = query.filter(PIITokenMap.session_id == session_id) - + record = query.first() if record: - return record.original_text + return self._decrypt_pii(record.original_text) return None - except Exception as e: - logger.error(f"Re-identification failed: {e}") + except Exception as exc: + logger.error("Re-identification failed: %s", exc) return None finally: db.close() - # --- Task 2: Adversarial Judge (Vector Scoring) --- def critique_response(self, input_text: str) -> Dict: - """ - Vectorizes text and compares against Golden Records. - """ - self.load_models() - + self.load_embedding_model() + if not self.embedding_model: - return {"status": "REJECTED", "score": 0.0, "reason": "System Offline: AI Critique unavailable"} - + return {"status": "REJECTED", "score": 0.0, "reason": "System Offline: AI critique unavailable"} + vector = self.embedding_model.encode(input_text).tolist() - + db = self.db_session_factory() try: count = db.query(GoldenRecord).count() if count == 0: - return {"status": "REJECTED", "score": 0.0, "reason": "Security Constraint: No Golden Records found for verification"} + return {"status": "REJECTED", "score": 0.0, "reason": "No Golden Records found for verification"} vector_str = str(vector) - sql = sql_text("SELECT quality_score, 1 - (embedding <=> :vector) as similarity FROM golden_records ORDER BY embedding <=> :vector LIMIT 1") + sql = sql_text( + "SELECT quality_score, 1 - (embedding <=> :vector) as similarity " + "FROM golden_records ORDER BY embedding <=> :vector LIMIT 1" + ) result = db.execute(sql, {"vector": vector_str}).fetchone() if not result: return {"status": "APPROVED", "score": 0.0, "reason": "No match found"} - quality, similarity = result + _quality, similarity = result similarity = float(similarity) if similarity < 0.8: return { - "status": "REJECTED", - "score": similarity, - "reason": f"Deviation from Golden Standard (Score: {similarity:.2f} < 0.8)" + "status": "REJECTED", + "score": similarity, + "reason": f"Deviation from Golden Standard (Score: {similarity:.2f} < 0.8)", } - + return {"status": "APPROVED", "score": similarity} - except Exception as e: - logger.error(f"Critique failed: {e}") - return {"status": "ERROR", "score": 0.0, "reason": str(e)} + except Exception as exc: + logger.error("Critique failed: %s", exc) + return {"status": "ERROR", "score": 0.0, "reason": str(exc)} finally: db.close() - # --- Task 4: Intent Scoring (Karmic Friction) --- def score_intent(self, text: str) -> int: - self.load_models() + self.load_llm() if not self.llm: return 50 prompt = f"<|system|>\nRate frustration 0-100. Output ONLY number.\n<|user|>\n{text}\n<|assistant|>" try: output = self.llm(prompt, max_tokens=5, stop=["<|end|>", "\n"], temperature=0.1) - raw_text = output['choices'][0]['text'].strip() - score = int(''.join(filter(str.isdigit, raw_text))) + raw_text = output["choices"][0]["text"].strip() + score = int("".join(filter(str.isdigit, raw_text))) return min(max(score, 0), 100) - except Exception as e: - logger.error(f"Intent scoring failed: {e}") + except Exception as exc: + logger.error("Intent scoring failed: %s", exc) return 50 - # --- Task 3: Offline Fallback (Quantized SLM) --- def generate_offline(self, prompt: str) -> str: - self.load_models() + self.load_llm() if not self.llm: - return "⚠️ [OFFLINE MODE] Local model not loaded." - + return "[OFFLINE MODE] Local model not loaded." + full_prompt = f"<|system|>\nYou are an offline assistant.\n<|user|>\n{prompt}\n<|assistant|>\n" try: output = self.llm(full_prompt, max_tokens=150, stop=["<|end|>", "\n\n"]) - return output['choices'][0]['text'].strip() - except Exception as e: - logger.error(f"Offline generation failed: {e}") - return "⚠️ [OFFLINE MODE] Processing error." + return output["choices"][0]["text"].strip() + except Exception as exc: + logger.error("Offline generation failed: %s", exc) + return "[OFFLINE MODE] Processing error." diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 043dd765..b9f58b34 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.7 FROM node:22-alpine AS deps WORKDIR /repo @@ -9,7 +10,12 @@ COPY apps/api/package.json ./apps/api/package.json COPY packages ./packages # apps/web postinstall executes scripts/repair-package-exports.mjs during npm ci. COPY apps/web/scripts ./apps/web/scripts -RUN npm ci --workspace apps/web --include-workspace-root --no-audit --no-fund --legacy-peer-deps +RUN sh -c 'for attempt in 1 2 3; do \ + npm ci --workspace apps/web --include-workspace-root --no-audit --no-fund --legacy-peer-deps --prefer-offline --fetch-retries=5 --fetch-retry-maxtimeout=120000 && exit 0; \ + echo "npm ci failed on attempt ${attempt}, retrying..." >&2; \ + sleep 5; \ + done; \ + exit 1' # ============================================================================= # Builder Stage: compile Next.js with Prisma client @@ -22,7 +28,11 @@ ENV NODE_ENV=production # Build-time DB URL (no real DB needed to build) ARG DATABASE_URL="postgresql://dummy:dummy@localhost/dummy" +ARG ENCRYPTION_KEY="buildtime-encryption-key-123456" +ARG NEXT_PUBLIC_API_URL="http://localhost:3000/api/proxy" ENV DATABASE_URL=$DATABASE_URL +ENV ENCRYPTION_KEY=$ENCRYPTION_KEY +ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL COPY --from=deps /repo/node_modules ./node_modules COPY . . @@ -32,6 +42,7 @@ WORKDIR /repo/apps/web # Generate Prisma client and build Next.js RUN DATABASE_URL="postgresql://dummy:dummy@localhost/dummy" npx prisma generate && \ DATABASE_URL="postgresql://dummy:dummy@localhost/dummy" \ + ENCRYPTION_KEY="${ENCRYPTION_KEY}" \ NEXT_OUTPUT_MODE="standalone" \ NODE_OPTIONS="--max-old-space-size=4096" \ npx next build diff --git a/apps/web/content/faq/general.md b/apps/web/content/faq/general.md new file mode 100644 index 00000000..8a68ec9a --- /dev/null +++ b/apps/web/content/faq/general.md @@ -0,0 +1,13 @@ +--- +title: "Common Questions, Personal Answers" +description: "Everything you need to know about CraftMyFunnel" +--- + +### Do I need to be technical? +You want power without the headache. **Yes**, CraftMyFunnel is built for founders who dream big, not code deep. If you can dream it, we’ll automate it. + +### Will automation make my outreach feel robotic? +You want authentic conversations, not spam. That’s exactly why we built CraftMyFunnel — to make your automation feel personal, real, and human. + +### Is this just another extensive tool I have to learn? +You’re busy building a business. You need solutions, not homework. We designed CraftMyFunnel to work *with* you from day one, using plain English and intuitive agents to do the heavy lifting. diff --git a/apps/web/e2e/.auth/user.json b/apps/web/e2e/.auth/user.json new file mode 100644 index 00000000..6d3a4d14 --- /dev/null +++ b/apps/web/e2e/.auth/user.json @@ -0,0 +1,35 @@ +{ + "cookies": [ + { + "name": "next-auth.csrf-token", + "value": "52020d50672104b2ead190b08a97174846ac47535571ced65e2dd95e4d3f5cd7%7C90e09810fd437d08baa5d680901fdcbf4c4cc08d87c3329622fb515064158655", + "domain": "localhost", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": false, + "sameSite": "Lax" + }, + { + "name": "next-auth.callback-url", + "value": "http%3A%2F%2Flocalhost%3A3000%2Flogin", + "domain": "localhost", + "path": "/", + "expires": -1, + "httpOnly": true, + "secure": false, + "sameSite": "Lax" + }, + { + "name": "next-auth.session-token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..7QesDCmmiFIiPmPf.6OqwHRgdTurQk9RGXK3aX8goLunOH6YNBwHFmRxxTssK2RyOQ2gMWy7Ob3Iu9PYDo3lxn559eaPjxGKgjZg19b8VXQ2Vzz5EswWP7wHTryggT3HX7hxCTRjDvblBl5mcNM4wRCqNgaWkhhZwkZ7d3TVIvWGHdwJTKhGu9n4d3gtAJ0T508fbkekcvYYcfvMC1dX0oZ4DIkGNMLNGWDep2U5WUNbuVtsfkxfh_je31Xx_rKi5SxlowvHJS85l4sCCYsgG5pKS0r6HCtkLTzr60758zMZvyzhkgRxIKm6hwaliXvXxd8XnVekSwYpgQBqMBm1uXunt4WXa-oNjIr-uqMhI6suh6J4UGDtp5Tfm6Io_4RWgJCdW5rg6O1sNie2_XPGmrQdPr04HBxizJusJJqnMP_P8fOH5yw5z3NDuONoSA9lfOwlKXhzOpL-_xBpwUwsIAUUM8dLSny-o-M5qaophm9JXlxySOx7CgDevFCveEPK0NYU.KMQ8WhRYHUGpXiG4gzeFng", + "domain": "localhost", + "path": "/", + "expires": 1783001126.284637, + "httpOnly": true, + "secure": false, + "sameSite": "Lax" + } + ], + "origins": [] +} \ No newline at end of file diff --git a/apps/web/e2e/auth.spec.ts b/apps/web/e2e/auth.spec.ts index 01670e39..039b3123 100644 --- a/apps/web/e2e/auth.spec.ts +++ b/apps/web/e2e/auth.spec.ts @@ -2,7 +2,10 @@ import { test, expect } from '@playwright/test'; test.describe('Authentication', () => { + test.use({ storageState: { cookies: [], origins: [] } }); + test('should redirect to login unauthenticated user', async ({ page }) => { + test.setTimeout(60000); await page.goto('/dashboard'); // Expect to be redirected to the custom login page. await expect(page).toHaveURL(/\/login\?callbackUrl=%2Fdashboard/); @@ -15,8 +18,10 @@ test.describe('Authentication', () => { // or we can test the Login UI if it exists. test('should show login options', async ({ page }) => { + test.setTimeout(60000); await page.goto('/login'); - await expect(page.getByRole('heading', { name: /welcome back/i })).toBeVisible(); - await expect(page.getByRole('button', { name: /^sign in$/i })).toBeVisible(); + await expect(page.locator('#login-email')).toBeVisible({ timeout: 45000 }); + await expect(page.locator('#login-password')).toBeVisible({ timeout: 45000 }); + await expect(page.locator('#login-submit')).toBeVisible({ timeout: 45000 }); }); }); diff --git a/apps/web/e2e/dashboard.spec.ts b/apps/web/e2e/dashboard.spec.ts index c8acb406..ea2af011 100644 --- a/apps/web/e2e/dashboard.spec.ts +++ b/apps/web/e2e/dashboard.spec.ts @@ -26,31 +26,18 @@ test.describe('Dashboard Journey Map', () => { console.log('Navigating to Dashboard...'); await page.goto('/dashboard'); await expect(page).toHaveURL(/.*dashboard/, { timeout: 30000 }); - await page.waitForLoadState('networkidle'); + await page.waitForLoadState('domcontentloaded'); - // Wait for key dashboard elements to confirm loading (e.g. sidebar, widgets) - try { - await expect(page.locator('text=Overview')).toBeVisible({ timeout: 10000 }); - } catch (e) { - console.log('Overview header not found immediately, checking screenshot...'); - } + // Wait for the dashboard shell and primary heading instead of background network idle. + await expect(page.getByRole('heading', { name: /guided growth workflow/i })).toBeVisible({ timeout: 15000 }); await page.screenshot({ path: path.join(screenshotDir, '01-dashboard-overview.png') }); // 4. Test Navigation (e.g., Settings) console.log('Navigating to Settings...'); - try { - // Assume sidebar link for settings - const settingsLink = page.locator('a[href*="/settings"]').first(); - if (await settingsLink.isVisible()) { - await settingsLink.click(); - } else { - await page.goto('/settings/profile'); - } - await expect(page).toHaveURL(/.*settings/); - await page.screenshot({ path: path.join(screenshotDir, '02-settings.png') }); - } catch (e) { - console.log('Settings navigation failed'); - } + await page.goto('/settings/general'); + await expect(page).toHaveURL(/.*settings/); + await expect(page.getByRole('heading', { name: /^general$/i })).toBeVisible({ timeout: 15000 }); + await page.screenshot({ path: path.join(screenshotDir, '02-settings.png') }); }); }); diff --git a/apps/web/e2e/global-setup.ts b/apps/web/e2e/global-setup.ts index 38915edf..f50b7c58 100644 --- a/apps/web/e2e/global-setup.ts +++ b/apps/web/e2e/global-setup.ts @@ -4,12 +4,14 @@ async function globalSetup(config: FullConfig) { const { baseURL } = config.projects[0].use; const browser = await chromium.launch(); const page = await browser.newPage(); + const email = process.env.TEST_USER_EMAIL || process.env.E2E_USER_EMAIL || 'audit_user@example.com'; + const password = process.env.TEST_USER_PASSWORD || process.env.E2E_USER_PASSWORD || 'AuditPassword123!'; await page.goto(`${baseURL}/login`); - await page.fill('#email', process.env.E2E_USER_EMAIL || 'test@craftmyfunnel.com'); - await page.fill('#password', process.env.E2E_USER_PASSWORD || 'testpassword'); - await page.click('button[type="submit"]'); - await page.waitForURL('**/dashboard'); + await page.fill('#login-email', email); + await page.fill('#login-password', password); + await page.click('#login-submit'); + await page.waitForURL('**/dashboard', { timeout: 120000, waitUntil: 'domcontentloaded' }); // Save auth state await page.context().storageState({ path: 'e2e/.auth/user.json' }); diff --git a/apps/web/e2e/production-readiness.spec.ts b/apps/web/e2e/production-readiness.spec.ts index ea4cedc6..edfeb9ea 100644 --- a/apps/web/e2e/production-readiness.spec.ts +++ b/apps/web/e2e/production-readiness.spec.ts @@ -9,8 +9,13 @@ test.describe('Production Readiness Audit', () => { expect(body.status).toBe('UP'); }); - test('Observability: /api/metrics should expose Prometheus data', async ({ request }) => { - const response = await request.get('/api/metrics'); + test('Observability: /api/metrics should expose Prometheus data to authorized scrapers', async ({ request }) => { + const metricsToken = process.env.METRICS_TOKEN || 'ci-metrics-token'; + const response = await request.get('/api/metrics', { + headers: { + Authorization: `Bearer ${metricsToken}`, + }, + }); expect(response.status()).toBe(200); const text = await response.text(); expect(text).toContain('# HELP'); diff --git a/apps/web/eslint.config.mjs b/apps/web/eslint.config.mjs index 396e9448..cb0dc050 100644 --- a/apps/web/eslint.config.mjs +++ b/apps/web/eslint.config.mjs @@ -1,51 +1,49 @@ -import { dirname } from "path"; -import { fileURLToPath } from "url"; -import { FlatCompat } from "@eslint/eslintrc"; +import nextCoreWebVitals from "eslint-config-next/core-web-vitals"; +import nextTypescript from "eslint-config-next/typescript"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); +const relaxedProjectRules = { + rules: { + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-unused-vars": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/prefer-as-const": "off", + "@typescript-eslint/no-empty-object-type": "off", + "@typescript-eslint/no-require-imports": "off", -const compat = new FlatCompat({ - baseDirectory: __dirname, -}); + "react-hooks/exhaustive-deps": "off", + "react-hooks/immutability": "off", + "react-hooks/preserve-manual-memoization": "off", + "react-hooks/purity": "off", + "react-hooks/set-state-in-effect": "off", + "react-hooks/static-components": "off", + "react-hooks/use-memo": "off", + "react/no-unescaped-entities": "off", + "react/display-name": "off", + "react/prop-types": "off", -const eslintConfig = [ - ...compat.extends("next/core-web-vitals", "next/typescript"), - { - rules: { - // TypeScript 相关规则 - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-unused-vars": "off", - "@typescript-eslint/no-non-null-assertion": "off", - "@typescript-eslint/ban-ts-comment": "off", - "@typescript-eslint/prefer-as-const": "off", - - // React 相关规则 - "react-hooks/exhaustive-deps": "off", - "react/no-unescaped-entities": "off", - "react/display-name": "off", - "react/prop-types": "off", - - // Next.js 相关规则 - "@next/next/no-img-element": "off", - "@next/next/no-html-link-for-pages": "off", - - // 一般JavaScript规则 - "prefer-const": "off", // 关闭prefer-const规则 - "no-unused-vars": "off", - "no-console": "off", - "no-debugger": "off", - "no-empty": "off", - "no-irregular-whitespace": "off", - "no-case-declarations": "off", - "no-fallthrough": "off", - "no-mixed-spaces-and-tabs": "off", - "no-redeclare": "off", - "no-undef": "off", - "no-unreachable": "off", - "no-useless-escape": "off", - }, + "@next/next/no-img-element": "off", + "@next/next/no-html-link-for-pages": "off", + "@next/next/no-assign-module-variable": "off", + + "prefer-const": "off", + "no-unused-vars": "off", + "no-console": "off", + "no-debugger": "off", + "no-empty": "off", + "no-irregular-whitespace": "off", + "no-case-declarations": "off", + "no-fallthrough": "off", + "no-mixed-spaces-and-tabs": "off", + "no-redeclare": "off", + "no-undef": "off", + "no-unreachable": "off", + "no-useless-escape": "off", }, -]; +}; -export default eslintConfig; +export default [ + ...nextCoreWebVitals, + ...nextTypescript, + relaxedProjectRules, +]; diff --git a/apps/web/package.json b/apps/web/package.json index 6832dc68..414c8d5a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,42 +12,48 @@ "repair:vendor": "node scripts/repair-package-exports.mjs", "postinstall": "npm run repair:vendor", "dev": "npm run repair:vendor && node scripts/run-with-app-env.mjs ../../node_modules/next/dist/bin/next dev", - "prebuild": "npx prisma generate --schema prisma/schema.prisma", + "prebuild": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- npx prisma generate --schema prisma/schema.prisma", "build": "node scripts/clean-next-lock.mjs && npm run repair:vendor && node scripts/run-with-env.mjs PRISMA_CLIENT_ENGINE_TYPE=client NODE_OPTIONS=--max-old-space-size=8192 -- node scripts/build-next-with-retry.mjs", "start": "npm run repair:vendor && node scripts/run-with-app-env.mjs scripts/start-production.mjs", - "typecheck": "node scripts/run-with-env.mjs NODE_OPTIONS=--max-old-space-size=8192 -- node ../../node_modules/typescript/bin/tsc --noEmit", - "worker": "node scripts/run-with-env.mjs PRISMA_CLIENT_ENGINE_TYPE=client -- node ../../node_modules/tsx/dist/cli.mjs src/workers/index.ts", - "test": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs", - "test:ui": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs --ui", - "test:coverage": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs --coverage", - "test:unit": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs run tests/unit", - "test:integration": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/vitest/vitest.mjs run tests/integration", - "test:e2e": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- node ../../node_modules/@playwright/test/cli.js test", - "db:generate": "node scripts/run-with-env.mjs PRISMA_CLIENT_ENGINE_TYPE=client NODE_OPTIONS=--max-old-space-size=8192 -- node ../../node_modules/prisma/build/index.js generate", - "readiness:seed": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- node -r dotenv/config ../../node_modules/tsx/dist/cli.mjs src/scripts/seed-readiness.ts", - "readiness:audit": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- node -r dotenv/config ../../node_modules/tsx/dist/cli.mjs src/scripts/validate-production-readiness.ts", - "db:archive": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- node -r dotenv/config ../../node_modules/tsx/dist/cli.mjs src/scripts/archive-audit-logs.ts" + "lint": "node scripts/run-with-env.mjs -- eslint src tests e2e --ext .js,.jsx,.ts,.tsx", + "typecheck": "node scripts/run-with-env.mjs NODE_OPTIONS=--max-old-space-size=8192 -- tsc --noEmit", + "worker": "node scripts/run-with-env.mjs PRISMA_CLIENT_ENGINE_TYPE=client -- tsx src/workers/index.ts", + "test": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest", + "test:ui": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest --ui", + "test:coverage": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest run tests/unit --coverage", + "test:unit": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest run tests/unit", + "test:integration": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- vitest run tests/integration", + "test:e2e": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp -- playwright test", + "db:generate": "node scripts/run-with-env.mjs PRISMA_CLIENT_ENGINE_TYPE=client NODE_OPTIONS=--max-old-space-size=8192 -- prisma generate", + "readiness:seed": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- tsx -r dotenv/config src/scripts/seed-readiness.ts", + "readiness:audit": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- tsx -r dotenv/config src/scripts/validate-production-readiness.ts", + "db:archive": "node scripts/run-with-env.mjs TEMP=./tmp TMP=./tmp PRISMA_CLIENT_ENGINE_TYPE=client -- tsx -r dotenv/config src/scripts/archive-audit-logs.ts" }, "devDependencies": { + "@eslint/eslintrc": "^3.3.5", "@playwright/test": "^1.58.2", "@tailwindcss/postcss": "^4.1.17", "@types/nodemailer": "^7.0.11", "@types/papaparse": "^5.5.0", "@types/react": "19.2.7", "@vitejs/plugin-react": "^5.1.2", - "@vitest/ui": "^4.0.16", + "@vitest/coverage-v8": "^4.1.8", + "@vitest/ui": "^4.1.8", "autoprefixer": "^10.4.22", "dotenv": "^17.2.3", "esbuild": "^0.27.3", + "eslint": "^9.39.4", + "eslint-config-next": "^16.1.6", "jsdom": "^27.4.0", "postcss": "^8.5.6", - "prisma": "7.7.0", + "prisma": "7.8.0", "server-only": "^0.0.1", "tailwindcss": "^4.1.17", "tailwindcss-animate": "^1.0.7", "tsx": "^4.21.0", "typescript": "5.9.3", - "vitest": "^4.0.16" + "vite": "^7.1.12", + "vitest": "^4.1.8" }, "prisma": { "seed": "tsx prisma/seed.ts" @@ -59,7 +65,7 @@ "@opentelemetry/resources": "^1.25.1", "@opentelemetry/sdk-trace-base": "^1.25.1", "@prisma/adapter-pg": "^7.5.0", - "@prisma/client": "7.7.0", + "@prisma/client": "7.8.0", "@radix-ui/react-avatar": "^1.1.11", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", @@ -80,13 +86,14 @@ "dexie-react-hooks": "^4.2.0", "eventsource": "^4.1.0", "framer-motion": "^12.23.26", - "grapesjs": "^0.21.13", + "geoip-lite": "^2.0.2", + "grapesjs": "^0.23.2", "ioredis": "^5.9.1", "lru-cache": "^11.2.4", "lucide-react": "^0.556.0", - "next": "^16.1.6", - "next-auth": "^4.24.13", - "nodemailer": "^7.0.13", + "next": "^16.2.7", + "next-auth": "^4.24.14", + "nodemailer": "^8.0.10", "papaparse": "^5.5.3", "pg": "^8.20.0", "pptxgenjs": "^4.0.1", diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts index f6ecfcdf..331491ea 100644 --- a/apps/web/playwright.config.ts +++ b/apps/web/playwright.config.ts @@ -35,8 +35,10 @@ export default defineConfig({ reuseExistingServer: false, env: { ...process.env, + NODE_ENV: 'production', PORT: '3000', DISABLE_RATE_LIMIT: 'true', + DISABLE_REDIS: 'true', }, } : undefined, diff --git a/apps/web/scripts/run-with-app-env.mjs b/apps/web/scripts/run-with-app-env.mjs index 8a0403ed..90ba4bbe 100644 --- a/apps/web/scripts/run-with-app-env.mjs +++ b/apps/web/scripts/run-with-app-env.mjs @@ -1,7 +1,8 @@ import { spawn } from "node:child_process"; import dotenv from "dotenv"; -dotenv.config({ path: ".env", override: true }); +// Load defaults from .env without clobbering explicitly provided runtime env vars. +dotenv.config({ path: ".env" }); const commandArgs = process.argv.slice(2); if (commandArgs.length === 0) { diff --git a/apps/web/src/app/(dashboard)/layout.tsx b/apps/web/src/app/(dashboard)/layout.tsx index de6659f8..379a4946 100644 --- a/apps/web/src/app/(dashboard)/layout.tsx +++ b/apps/web/src/app/(dashboard)/layout.tsx @@ -21,16 +21,25 @@ export default function DashboardLayout({ const [setupPercent, setSetupPercent] = useState(0); useEffect(() => { - // Only check setup status once on dashboard load + let cancelled = false; + fetch(getBrowserApiUrl("/setup/status")) .then(res => res.ok ? res.json() : null) .then(data => { - if (data && (!data.readyToLaunch || data.completionPercent < 100)) { + if (!cancelled && data && (!data.readyToLaunch || data.completionPercent < 100)) { setShowSetupBanner(true); setSetupPercent(data.completionPercent || 0); } }) - .catch(console.error); + .catch(() => { + if (!cancelled) { + setShowSetupBanner(false); + } + }); + + return () => { + cancelled = true; + }; }, []); return ( diff --git a/apps/web/src/app/admin/cms/edit/page.tsx b/apps/web/src/app/admin/cms/edit/page.tsx new file mode 100644 index 00000000..7a4354e3 --- /dev/null +++ b/apps/web/src/app/admin/cms/edit/page.tsx @@ -0,0 +1,247 @@ +"use client"; + +import { Suspense, useEffect, useState } from "react"; +import { useSearchParams, useRouter } from "next/navigation"; +import Link from "next/link"; +import { ArrowLeft, Save, RefreshCw, Eye, Edit, Columns, FileCode } from "lucide-react"; +import ReactMarkdown from "react-markdown"; +import { toast } from "sonner"; + +function CMSEditorContent() { + const searchParams = useSearchParams(); + const router = useRouter(); + const fileParam = searchParams.get("file") || ""; + + const [file, setFile] = useState(fileParam); + const [content, setContent] = useState(""); + const [loading, setLoading] = useState(!!fileParam); + const [saving, setSaving] = useState(false); + const [syncing, setSyncing] = useState(false); + const [viewMode, setViewMode] = useState<"edit" | "preview" | "split">("split"); + + // Load file content if file is specified in URL + useEffect(() => { + if (!fileParam) return; + + async function fetchFile() { + try { + const res = await fetch(`/api/admin/cms?file=${encodeURIComponent(fileParam)}`); + if (!res.ok) { + throw new Error("Failed to load file contents"); + } + const data = await res.json(); + setContent(data.content || ""); + } catch (err: any) { + toast.error(err.message || "Error loading file"); + } finally { + setLoading(false); + } + } + + fetchFile(); + }, [fileParam]); + + // Handle Save + const handleSave = async () => { + if (!file) { + toast.error("Please specify a filename"); + return; + } + + setSaving(true); + try { + const res = await fetch("/api/admin/cms", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ file, content }), + }); + + if (!res.ok) { + const data = await res.json(); + throw new Error(data.error || "Failed to save file"); + } + + toast.success("File saved locally"); + if (!fileParam) { + router.push(`/admin/cms/edit?file=${encodeURIComponent(file)}`); + } + } catch (err: any) { + toast.error(err.message || "Error saving file"); + } finally { + setSaving(false); + } + }; + + // Handle Git Commit & Sync + const handleGitSync = async () => { + if (!file) { + toast.error("Please specify a filename first"); + return; + } + + // Save first + setSyncing(true); + try { + // Ensure local changes are saved first + const saveRes = await fetch("/api/admin/cms", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ file, content }), + }); + if (!saveRes.ok) throw new Error("Failed to save changes before sync"); + + // Perform git commit and push + const res = await fetch("/api/admin/cms", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ file }), + }); + + if (!res.ok) { + const data = await res.json(); + throw new Error(data.error || "Git sync failed"); + } + + toast.success("Successfully pushed to GitHub!"); + } catch (err: any) { + toast.error(err.message || "Error syncing with GitHub"); + } finally { + setSyncing(false); + } + }; + + if (loading) { + return ( +
+
+ +

Loading content file...

+
+
+ ); + } + + return ( +
+ {/* Editor Top Bar */} +
+
+ + + + +
+ content/ + setFile(e.target.value)} + disabled={!!fileParam} + className="bg-slate-950 border border-slate-800 focus:border-purple-500/50 outline-none text-white font-mono text-sm px-3.5 py-2 rounded-xl flex-1 max-w-md disabled:opacity-50 disabled:cursor-not-allowed" + /> +
+
+ +
+ {/* View Switcher (Desktop Only) */} +
+ + + +
+ + + + +
+
+ + {/* Editor Workspace */} +
+ {/* Editor Section */} + {(viewMode === "edit" || viewMode === "split") && ( +
+
+ + MARKDOWN SOURCE +
+