diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 8dc0259e..8f4b02c5 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -17,6 +17,9 @@ jobs: validate: uses: LerianStudio/github-actions-shared-workflows/.github/workflows/go-pr-validation.yml@v1.36.5 with: + enforce_source_branches: true + allowed_source_branches: 'hotfix/*|release-candidate' + target_branches_for_source_check: 'main' pr_title_scopes: | manager worker diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed3522a2..cbf7e52c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ permissions: jobs: pipeline: - uses: LerianStudio/github-actions-shared-workflows/.github/workflows/go-release.yml@v1.36.5 + uses: LerianStudio/github-actions-shared-workflows/.github/workflows/go-release.yml@v1.40.3 with: enable_changelog: ${{ github.ref == 'refs/heads/main' }} release_single_app: true diff --git a/CLAUDE.md b/CLAUDE.md index f3d00947..7b3e2012 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ Lerian Fetcher is an enterprise data extraction platform that unifies access to ```bash make set-env # Copy .env.example to .env for all components make generate-master-key # Generate APP_ENC_KEY (REQUIRED before services start) -make dev-setup # Install tools (golangci-lint, swag, mockgen, gosec) + tidy +make dev-setup # Install tools (golangci-lint, mockgen, gosec) + tidy ``` ### Build & Run @@ -44,7 +44,8 @@ make lint # golangci-lint with --fix (28 linters, max cyclomatic compl make format # go fmt make tidy # go mod tidy make sec # gosec security analysis -make generate-docs # Regenerate Swagger docs (swag init) +make generate-docs # Regenerate the canonical Huma OpenAPI 3.1 specification +make check-openapi # Fail when the committed OpenAPI specification is stale ``` ## Architecture @@ -115,7 +116,7 @@ Async RabbitMQ consumer. Does NOT follow CQRS - uses a single `UseCase` struct i **CQRS services (Manager only):** Struct with `Execute()` method. Constructor `NewXxx(deps...)`. Dependencies are port interfaces, never concrete implementations. -**HTTP handlers:** Thin handlers in `internal/adapters/http/in/` with Swagger annotations (swaggo). Extract org ID via `httpUtils.GetOrganizationID(c)`, product name from `X-Product-Name` header. +**HTTP operations:** Typed Huma callbacks in `internal/adapters/http/in/huma_*.go`, registered through the single runtime/spec assembly. Preserve the request context installed by auth/tenant middleware and read the product name from the `X-Product-Name` header when required. Return errors through the lib-commons RFC 9457 problem model. **Error handling:** Use `pkg.ValidateBusinessError(constant.ErrXxx, "entityType", args...)` for business errors. Custom types in `pkg/errors.go` map to HTTP status codes. @@ -165,7 +166,7 @@ Async RabbitMQ consumer. Does NOT follow CQRS - uses a single `UseCase` struct i | `components/manager/README.md` | Manager service (HTTP control plane, runs over the engine) | | `components/worker/README.md` | Worker service (async data plane, runs over the engine) | | `tests/e2e/README.md` | E2E test conventions, patterns, test catalog | -| `components/manager/api/swagger.yaml` | OpenAPI specification | +| `components/manager/api/openapi.yaml` | Canonical OpenAPI 3.1 specification | | `components/manager/api/requests.http` | API request examples for manual testing | | `.golangci.yml` | Full linter configuration (28 linters) | @@ -173,11 +174,11 @@ Async RabbitMQ consumer. Does NOT follow CQRS - uses a single `UseCase` struct i ### Adding a new API endpoint -1. Create handler in `components/manager/internal/adapters/http/in/` with Swagger annotations -2. Register route in the routes file +1. Add the typed callback and operation metadata in the appropriate `components/manager/internal/adapters/http/in/huma_*.go` file +2. Register the operation through the shared Huma assembly in `huma_operations.go` 3. Create command or query service in `internal/services/command/` or `internal/services/query/` 4. Wire dependencies in `internal/bootstrap/` -5. Run `make generate-docs` +5. Run `make generate-docs` and `make check-openapi` ### Adding a new database adapter diff --git a/Makefile b/Makefile index b0554ba5..3c4023d4 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,8 @@ help: @echo "" @echo "" @echo "Documentation Commands:" - @echo " make generate-docs - Generate Swagger documentation" + @echo " make generate-docs - Generate the canonical Huma OpenAPI 3.1 specification" + @echo " make check-openapi - Fail when the committed OpenAPI specification is stale" @echo "" @echo "" @echo "Test Suite Aliases:" @@ -230,8 +231,8 @@ set-env: # Deterministic, non-live quality gate. Mirrors the Lerian CI pattern # (cf. plugin-br-bank-transfer) restricted to steps that need no Docker # daemon, no live infrastructure, and no GITHUB_TOKEN — so it runs the -# same locally and in CI. Docker-bound steps (generate-docs, test-e2e, -# test-chaos) stay available standalone and run in their own CI stages. +# same locally and in CI. Docker-bound steps (test-e2e, test-chaos) stay +# available standalone and run in their own CI stages. .PHONY: ci ci: $(call print_title,Running full CI pipeline) @@ -240,6 +241,7 @@ ci: @$(MAKE) vet @$(MAKE) lint @$(MAKE) sec + @$(MAKE) check-openapi @$(MAKE) test-unit @echo "[ok] CI pipeline passed" @@ -795,20 +797,21 @@ ps: .PHONY: generate-docs generate-docs: - $(call print_title,Generating Swagger API documentation) - @if ! command -v swag >/dev/null 2>&1; then \ - echo "Installing swag..."; \ - go install github.com/swaggo/swag/cmd/swag@latest; \ - fi - @swag init -g ./components/manager/cmd/app/main.go -d ./ -o ./components/manager/api --parseDependency --parseInternal - @docker run --rm -v $(ROOT_DIR):/local --user $(shell id -u):$(shell id -g) openapitools/openapi-generator-cli:v5.1.1 generate -i /local/components/manager/api/swagger.json -g openapi-yaml -o /local/components/manager/api - @mv ./components/manager/api/openapi/openapi.yaml ./components/manager/api/openapi.yaml - @rm -rf ./components/manager/api/README.md ./components/manager/api/.openapi-generator* ./components/manager/api/openapi - @if [ -f "$(ROOT_DIR)/scripts/package.json" ]; then \ - echo "Installing npm dependencies for validation..."; \ - cd $(ROOT_DIR)/scripts && npm install > /dev/null; \ - fi - @echo "[ok] Swagger API documentation generated successfully" + $(call print_title,Generating canonical Huma OpenAPI specification) + @go run ./components/manager/cmd/huma-spec -output ./components/manager/api/openapi.yaml + @echo "[ok] OpenAPI 3.1 specification generated successfully" + +.PHONY: check-openapi +check-openapi: + $(call print_title,Checking committed OpenAPI specification) + @tmp=$$(mktemp); \ + trap 'rm -f "$$tmp"' EXIT; \ + go run ./components/manager/cmd/huma-spec -output "$$tmp" || { echo "OpenAPI specification generation failed"; exit 1; }; \ + cmp -s "$$tmp" ./components/manager/api/openapi.yaml || { \ + echo "OpenAPI specification is stale; run 'make generate-docs'"; \ + exit 1; \ + } + @echo "[ok] Committed OpenAPI specification is current" #------------------------------------------------------- # Developer Helper Commands @@ -822,10 +825,6 @@ dev-setup: echo "Installing golangci-lint..."; \ go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \ fi - @if ! command -v swag >/dev/null 2>&1; then \ - echo "Installing swag..."; \ - go install github.com/swaggo/swag/cmd/swag@latest; \ - fi @if ! command -v mockgen >/dev/null 2>&1; then \ echo "Installing mockgen..."; \ go install go.uber.org/mock/mockgen@latest; \ @@ -1020,6 +1019,3 @@ generate-master-key: ## Generate a new cryptographically secure master key echo "" && \ echo "IMPORTANT: Store this key securely. It cannot be recovered if lost." && \ echo "Add to your environment as APP_ENC_KEY." - - - diff --git a/README.md b/README.md index 5f16fab0..4ef4a2ff 100644 --- a/README.md +++ b/README.md @@ -232,14 +232,15 @@ Lerian Fetcher is built as a cloud-native platform following Hexagonal Architect | `GET` | `/health` | Liveness check | | `GET` | `/readyz` | Readiness check — parallel dependency probes (MongoDB, RabbitMQ, Redis; S3 on Worker), returns 503 while draining on SIGTERM | | `GET` | `/version` | Version info | -| `GET` | `/swagger/*` | Swagger UI | +| `GET` | `/swagger/docs` | Scalar API reference (`SWAGGER_ENABLED=true`) | +| `GET` | `/swagger/openapi.{json,yaml}` | OpenAPI 3.1 contract (`SWAGGER_ENABLED=true`) | ### API Reference & Testing For hands-on API exploration and testing scenarios, the following resources are available: - **[`components/manager/api/requests.http`](components/manager/api/requests.http)**: Ready-to-use HTTP request examples covering all API endpoints — useful for quick testing with VS Code REST Client, IntelliJ, or similar tools. -- **[`components/manager/api/swagger.yaml`](components/manager/api/swagger.yaml)**: Full OpenAPI specification for the Manager API, which can be imported into Postman, Insomnia, or any OpenAPI-compatible tool. +- **[`components/manager/api/openapi.yaml`](components/manager/api/openapi.yaml)**: Canonical OpenAPI 3.1 specification for the Manager API, generated from the same Huma assembly used at runtime. - **[`tests/e2e/`](tests/e2e/)**: End-to-end test suite covering connection management, data extraction across all supported databases, filtering, multi-datasource/multi-schema scenarios, schema validation, and error handling. These tests serve as practical usage examples and can be referenced to understand expected behaviors and edge cases. ### Technical Highlights @@ -315,7 +316,7 @@ Single-tenant deployments emit with stable tenant ID `single-tenant`; multi-tena 5. **Access the API:** - REST API: `http://localhost:4006` - - Swagger UI: `http://localhost:4006/swagger/index.html` + - Scalar API reference (when enabled): `http://localhost:4006/swagger/docs` - RabbitMQ Management: `http://localhost:3008` ### Security diff --git a/components/manager/.env.example b/components/manager/.env.example index b62a7fd9..3ccadb96 100644 --- a/components/manager/.env.example +++ b/components/manager/.env.example @@ -38,14 +38,7 @@ RABBITMQ_HEALTH_CHECK_URL=http://${RABBITMQ_HOST}:${RABBITMQ_PORT_HOST} #RABBITMQ_TLS=false # SWAGGER -SWAGGER_TITLE='Fetcher' -SWAGGER_DESCRIPTION='Documentation for fetcher' -SWAGGER_VERSION=${VERSION} -SWAGGER_HOST=${SERVER_ADDRESS} -SWAGGER_BASE_PATH=/ -SWAGGER_SCHEMES=http -SWAGGER_LEFT_DELIM={{ -SWAGGER_RIGHT_DELIM=}} +SWAGGER_ENABLED=false # AUTHORIZATION PLUGIN_AUTH_ENABLED=false @@ -87,6 +80,13 @@ MULTI_TENANT_ENABLED=false #MULTI_TENANT_CIRCUIT_BREAKER_THRESHOLD=5 #MULTI_TENANT_CIRCUIT_BREAKER_TIMEOUT_SEC=30 #MULTI_TENANT_SERVICE_API_KEY= +# Per-service tenant-manager API keys (optional). Scheme: +# MULTI_TENANT_SERVICE_API_KEY_ where = the service name, hyphens->underscores, upper-cased. +# When set, the resolver authenticates that service's tenant-config lookups with the matching key; +# falls back to MULTI_TENANT_SERVICE_API_KEY when a per-service key is absent. +# Adding a new service is env-only (no code change). Two env vars normalizing to the same token fail fast at startup. +#MULTI_TENANT_SERVICE_API_KEY_LEDGER= +#MULTI_TENANT_SERVICE_API_KEY_PLUGIN_CRM= #MULTI_TENANT_CACHE_TTL_SEC=120 #MULTI_TENANT_TIMEOUT=30 #MULTI_TENANT_ALLOW_INSECURE_HTTP=false diff --git a/components/manager/Dockerfile b/components/manager/Dockerfile index 7d9212e2..51fe7530 100644 --- a/components/manager/Dockerfile +++ b/components/manager/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 -FROM golang:1.26.4-alpine AS builder +FROM golang:1.26.5-alpine AS builder # Install required packages: git is needed for private modules RUN apk add --no-cache git diff --git a/components/manager/Makefile b/components/manager/Makefile index d8552e3a..77672072 100644 --- a/components/manager/Makefile +++ b/components/manager/Makefile @@ -47,7 +47,8 @@ info: @echo " " @echo "## App commands:" @echo " " - @echo " COMMAND=\"generate-docs\" Generates Swagger API documentation and an OpenAPI Specification." + @echo " COMMAND=\"generate-docs\" Generates the canonical Huma OpenAPI 3.1 specification." + @echo " COMMAND=\"check-openapi\" Fails when the committed OpenAPI specification is stale." @echo " " @echo " " @@ -85,10 +86,11 @@ ps: .PHONY: generate-docs generate-docs: - @swag init -g ./cmd/app/main.go -d ./ -o ./api --parseDependency --parseInternal - @docker run --rm -v $(pwd):/local --user $(shell id -u):$(shell id -g) openapitools/openapi-generator-cli:v5.1.1 generate -i ./manager/api/swagger.json -g openapi-yaml -o ./manager/api - @mv ./api/openapi/openapi.yaml ./api/openapi.yaml - @rm -rf ./api/README.md ./api/.openapi-generator* ./api/openapi + @$(MAKE) -C ../.. generate-docs + +.PHONY: check-openapi +check-openapi: + @$(MAKE) -C ../.. check-openapi .PHONY: setup-git-hooks setup-git-hooks: @@ -149,4 +151,4 @@ test: set-env: @echo "$(BLUE)Setting up environment files...$(NC)" cp -r ./.env.example ./.env - @echo "$(BLUE)Environment files created successfully$(NC)" \ No newline at end of file + @echo "$(BLUE)Environment files created successfully$(NC)" diff --git a/components/manager/README.md b/components/manager/README.md index 62f82fe9..ede15cec 100644 --- a/components/manager/README.md +++ b/components/manager/README.md @@ -2,7 +2,7 @@ The Manager is the Fetcher HTTP API server. It runs the synchronous control plane: connection CRUD, schema discovery and validation, connection testing, idempotency, rate limiting, license enforcement, and job dispatch onto RabbitMQ. The Worker does the actual extraction asynchronously. -The Manager is built on [Fiber](https://gofiber.io) and serves on port `4006`. It follows Hexagonal Architecture with CQRS: commands mutate state, queries read it, and both depend only on port interfaces. +The Manager is built on [Fiber](https://gofiber.io) with typed [Huma](https://huma.rocks) operations and serves on port `4006`. The same assembly drives runtime routing and the canonical OpenAPI 3.1 contract. It follows Hexagonal Architecture with CQRS: commands mutate state, queries read it, and both depend only on port interfaces. ## Table of Contents @@ -37,7 +37,7 @@ The Manager wires **two** engine instances, each a focused authority: ```mermaid graph TB - subgraph Manager["Manager (Fiber, port 4006)"] + subgraph Manager["Manager (Fiber + Huma, port 4006)"] Handlers["HTTP handlers\ninternal/adapters/http/in"] CmdQ["CQRS commands & queries\ninternal/services"] ConnEng["connection engine\nbootstrap/connection_engine.go"] @@ -121,7 +121,7 @@ graph LR Qry -. delegates rules .-> Engine ``` -- **Inbound adapters** (`internal/adapters/http/in/`) are thin Fiber handlers with Swagger annotations. They extract the organization ID via `httpUtils.GetOrganizationID(c)` and the product name from the `X-Product-Name` header, then call a service. +- **Inbound adapters** (`internal/adapters/http/in/`) expose typed Huma callbacks over Fiber. Operation metadata, request/response schemas, and RFC 9457 errors come from the same assembly used to generate OpenAPI 3.1. The callbacks preserve the auth/tenant request context, read the `X-Product-Name` header when required, then call a service. - **Commands** (`internal/services/command/`) handle connection Create, Update, and Delete, plus job creation (`CreateFetcherJob`) and connection assignment (`AssignConnection`). - **Queries** (`internal/services/query/`) handle Get, List, Test, and Validate. - **Bootstrap** (`internal/bootstrap/`) wires dependencies, including the two engines, and holds configuration. @@ -191,14 +191,14 @@ License enforcement is **fail-closed**: when `DEPLOYMENT_MODE` is not `local`, t | Path | Contents | |------|----------| | `cmd/app/main.go` | Entry point | -| `internal/adapters/http/in/` | Fiber HTTP handlers (with Swagger annotations) | +| `internal/adapters/http/in/` | Typed Huma operations and Fiber integration | | `internal/services/command/` | CQRS commands (Create, Update, Delete) | | `internal/services/query/` | CQRS queries (Get, List, Test, Validate) | | `internal/adapters/cache/` | Redis schema cache adapter | | `internal/bootstrap/` | Dependency injection, config, and engine wiring | | `internal/bootstrap/connection_engine.go` | Connection engine wiring | | `internal/bootstrap/schema_engine.go` | Schema engine wiring | -| `api/swagger.yaml` | OpenAPI specification | +| `api/openapi.yaml` | Canonical OpenAPI 3.1 specification | | `api/requests.http` | Manual API request examples | ## Configuration @@ -218,3 +218,5 @@ make generate-master-key | other non-`local` | Enforced (fail-closed) | — | See `internal/bootstrap/config.go` for the full configuration surface and `../../README.md` for the project-wide quick start. + +Set `SWAGGER_ENABLED=true` to expose the Scalar reference at `/swagger/docs` and the OpenAPI 3.1 contract at `/swagger/openapi.json` or `/swagger/openapi.yaml`. The routes are absent when the flag is disabled. diff --git a/components/manager/api/docs.go b/components/manager/api/docs.go deleted file mode 100644 index 06ce453f..00000000 --- a/components/manager/api/docs.go +++ /dev/null @@ -1,1401 +0,0 @@ -// Package api Code generated by swaggo/swag. DO NOT EDIT -package api - -import "github.com/swaggo/swag" - -const docTemplate = `{ - "schemes": {{ marshal .Schemes }}, - "swagger": "2.0", - "info": { - "description": "{{escape .Description}}", - "title": "{{.Title}}", - "termsOfService": "http://swagger.io/terms/", - "contact": {}, - "version": "{{.Version}}" - }, - "host": "{{.Host}}", - "basePath": "{{.BasePath}}", - "paths": { - "/v1/fetcher": { - "post": { - "description": "Create a new data extraction job. The request will be validated, deduplicated within a 5-minute window, and all referenced connections will be tested before job creation. The metadata.source field is required for product isolation and datasource ownership validation.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Fetcher" - ], - "summary": "Create fetcher job", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "description": "Fetcher request payload. metadata.source is required.", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherRequest" - } - } - ], - "responses": { - "200": { - "description": "Duplicate request - returning existing job", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse" - } - }, - "202": { - "description": "Job created and queued for processing", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "413": { - "description": "Request Entity Too Large", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/fetcher/{id}": { - "get": { - "description": "Retrieve detailed information about a specific data extraction job.", - "produces": [ - "application/json" - ], - "tags": [ - "Fetcher" - ], - "summary": "Get fetcher job", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Job ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.JobResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections": { - "get": { - "description": "List connections with pagination and filters. When X-Product-Name is provided, returns only connections associated with that product.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "List connections", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Product name. When provided, filters connections by product.", - "name": "X-Product-Name", - "in": "header" - }, - { - "type": "integer", - "default": 1, - "description": "Page number (minimum 1)", - "name": "page", - "in": "query" - }, - { - "type": "integer", - "default": 50, - "description": "Page size (default 50, max 1000)", - "name": "limit", - "in": "query" - }, - { - "enum": [ - "asc", - "desc" - ], - "type": "string", - "default": "desc", - "description": "Sort order", - "name": "sortOrder", - "in": "query" - }, - { - "enum": [ - "ORACLE", - "SQL_SERVER", - "POSTGRESQL", - "MONGODB", - "MYSQL" - ], - "type": "string", - "description": "Filter by database type", - "name": "type", - "in": "query" - }, - { - "type": "string", - "description": "Filter by host", - "name": "host", - "in": "query" - }, - { - "type": "string", - "description": "Filter by database name", - "name": "databaseName", - "in": "query" - }, - { - "type": "string", - "description": "Filter by start date (YYYY-MM-DD)", - "name": "startDate", - "in": "query" - }, - { - "type": "string", - "description": "Filter by end date (YYYY-MM-DD)", - "name": "endDate", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "allOf": [ - { - "$ref": "#/definitions/Pagination" - }, - { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "limit": { - "type": "integer" - }, - "page": { - "type": "integer" - }, - "total": { - "type": "integer" - } - } - } - ] - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - }, - "post": { - "description": "Create a new database connection for the organization. The X-Product-Name header is required and identifies the product for this connection. ConfigName must be unique per organization (and per product when assigned); password is encrypted and a UUID is generated on creation.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Create connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Product name (required, non-empty)", - "name": "X-Product-Name", - "in": "header", - "required": true - }, - { - "description": "Connection payload", - "name": "connection", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionInput" - } - } - ], - "responses": { - "201": { - "description": "Created connection identifier", - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/unassigned": { - "get": { - "description": "List connections that have no product assigned, useful for migration purposes.", - "produces": [ - "application/json" - ], - "tags": [ - "Migration" - ], - "summary": "List unassigned connections", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "integer", - "default": 1, - "description": "Page number (minimum 1)", - "name": "page", - "in": "query" - }, - { - "type": "integer", - "default": 50, - "description": "Page size (default 50, max 1000)", - "name": "limit", - "in": "query" - }, - { - "enum": [ - "asc", - "desc" - ], - "type": "string", - "default": "desc", - "description": "Sort order", - "name": "sortOrder", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "allOf": [ - { - "$ref": "#/definitions/Pagination" - }, - { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "limit": { - "type": "integer" - }, - "page": { - "type": "integer" - }, - "total": { - "type": "integer" - } - } - } - ] - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/validate-schema": { - "post": { - "description": "Validate that tables and fields referenced in the request exist in the configured datasources. Returns 200 when validation passes, 422 when validation fails with detailed error information.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Validate schema", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "description": "Schema validation request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest" - } - } - ], - "responses": { - "200": { - "description": "Validation successful - all tables and fields exist", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse" - } - }, - "400": { - "description": "Invalid request payload or missing headers", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "422": { - "description": "Validation failed - schema errors found (missing tables, fields, or unreachable datasources)", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse" - } - }, - "500": { - "description": "Internal server error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}": { - "get": { - "description": "Get connection details by ID for the given organization.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Get connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - }, - "delete": { - "description": "Soft delete a connection when no active jobs are running for it. Returns 409 if there is any active job.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Delete connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "No Content" - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - }, - "patch": { - "description": "Apply a partial update to a connection. Only include fields you want to change. Returns 409 if there is any active job.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Update connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "Fields to update (only include fields you want to change)", - "name": "connection", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}/assign": { - "post": { - "description": "Associate an unassigned connection to a product. This is a one-time, irreversible operation for migration purposes. The product name must be provided via the X-Product-Name header.", - "produces": [ - "application/json" - ], - "tags": [ - "Migration" - ], - "summary": "Assign connection to product", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Product name to assign", - "name": "X-Product-Name", - "in": "header", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}/schema": { - "get": { - "description": "Get the database schema (tables and fields) for a connection.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Get connection schema", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}/test": { - "post": { - "description": "Test the configured connection by establishing and closing a connection.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Test connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Connection test result", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "429": { - "description": "Too Many Requests", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - } - }, - "definitions": { - "Pagination": { - "description": "Pagination is the struct designed to store the pagination data of an entity list.", - "type": "object", - "properties": { - "items": {}, - "limit": { - "type": "integer", - "example": 10 - }, - "page": { - "type": "integer", - "example": 1 - }, - "total": { - "type": "integer", - "example": 10 - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionInput": { - "type": "object", - "required": [ - "configName", - "databaseName", - "host", - "password", - "port", - "type", - "userName" - ], - "properties": { - "configName": { - "type": "string", - "maxLength": 100, - "minLength": 3, - "example": "production-db" - }, - "databaseName": { - "type": "string", - "example": "mydatabase" - }, - "host": { - "type": "string", - "example": "db.example.com" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "password": { - "type": "string", - "example": "secretpassword" - }, - "port": { - "type": "integer", - "maximum": 65535, - "minimum": 1, - "example": 5432 - }, - "schema": { - "type": "string", - "example": "my_schema" - }, - "ssl": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLInput" - }, - "type": { - "type": "string", - "enum": [ - "ORACLE", - "SQL_SERVER", - "POSTGRESQL", - "MONGODB", - "MYSQL" - ], - "example": "POSTGRESQL" - }, - "userName": { - "type": "string", - "example": "dbuser" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse": { - "type": "object", - "properties": { - "configName": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "host": { - "type": "string" - }, - "id": { - "type": "string" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "port": { - "type": "integer" - }, - "productName": { - "type": "string" - }, - "schema": { - "type": "string" - }, - "ssl": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLResponse" - }, - "type": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "userName": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse": { - "type": "object", - "properties": { - "configName": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "id": { - "type": "string" - }, - "tables": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.TableDetails" - } - }, - "type": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput": { - "type": "object", - "properties": { - "configName": { - "type": "string", - "maxLength": 100, - "minLength": 3, - "example": "production-db" - }, - "databaseName": { - "type": "string", - "example": "mydatabase" - }, - "host": { - "type": "string", - "example": "db.example.com" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "password": { - "type": "string", - "example": "secretpassword" - }, - "port": { - "type": "integer", - "maximum": 65535, - "minimum": 1, - "example": 5432 - }, - "schema": { - "type": "string", - "example": "my_schema" - }, - "ssl": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput" - }, - "type": { - "type": "string", - "enum": [ - "ORACLE", - "SQL_SERVER", - "POSTGRESQL", - "MONGODB", - "MYSQL" - ], - "example": "POSTGRESQL" - }, - "userName": { - "type": "string", - "example": "dbuser" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.DataRequest": { - "description": "DataRequest encapsulates field mappings and optional filters for data extraction.", - "type": "object", - "required": [ - "mappedFields" - ], - "properties": { - "filters": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.NestedFilters" - }, - "mappedFields": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.FetcherRequest": { - "description": "FetcherRequest represents the request body for creating a new data extraction job.", - "type": "object", - "required": [ - "dataRequest" - ], - "properties": { - "dataRequest": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.DataRequest" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.FetcherResponse": { - "description": "FetcherResponse represents the response after successfully creating a data extraction job.", - "type": "object", - "properties": { - "createdAt": { - "type": "string" - }, - "jobId": { - "type": "string" - }, - "message": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.JobResponse": { - "description": "JobResponse represents the complete information about a data extraction job.", - "type": "object", - "properties": { - "completedAt": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "filters": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.NestedFilters" - }, - "id": { - "type": "string" - }, - "mappedFields": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "requestHash": { - "type": "string" - }, - "resultHmac": { - "type": "string" - }, - "resultPath": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.NestedFilters": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition" - } - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SSLInput": { - "type": "object", - "properties": { - "ca": { - "type": "string", - "example": "-----BEGIN CERTIFICATE-----\n..." - }, - "cert": { - "type": "string" - }, - "key": { - "type": "string" - }, - "mode": { - "type": "string", - "example": "require" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SSLResponse": { - "type": "object", - "properties": { - "mode": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput": { - "type": "object", - "properties": { - "ca": { - "type": "string", - "example": "-----BEGIN CERTIFICATE-----\n..." - }, - "cert": { - "type": "string" - }, - "key": { - "type": "string" - }, - "mode": { - "type": "string", - "example": "require" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError": { - "type": "object", - "properties": { - "dataSourceId": { - "type": "string" - }, - "field": { - "type": "string" - }, - "table": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse": { - "type": "object", - "properties": { - "code": { - "type": "string" - }, - "errors": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError" - } - }, - "message": { - "type": "string" - }, - "title": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest": { - "description": "Request body for schema validation containing mapped fields per datasource.", - "type": "object", - "required": [ - "mappedFields" - ], - "properties": { - "mappedFields": { - "description": "MappedFields maps datasource config names to their tables and fields\nKey: configName (e.g., \"midaz_onboarding\")\nValue: map of table names to field names", - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse": { - "type": "object", - "properties": { - "errors": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError" - } - }, - "message": { - "type": "string" - }, - "status": { - "description": "\"success\" or \"failure\"", - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.TableDetails": { - "type": "object", - "properties": { - "fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition": { - "type": "object", - "properties": { - "between": { - "description": "Between specifies a range condition with exactly two values [min, max].\nMatches records where min \u003c= field \u003c= max\nExample: {\"between\": [100, 1000]} matches records where 100 \u003c= field \u003c= 1000", - "type": "array", - "items": {} - }, - "eq": { - "description": "Equals specifies exact value matches. Multiple values treated as OR conditions.\nExample: {\"eq\": [\"active\", \"pending\"]} matches records where field equals \"active\" OR \"pending\"", - "type": "array", - "items": {} - }, - "gt": { - "description": "GreaterThan specifies values that must be greater than the provided value.\nShould contain exactly one value for comparison.\nExample: {\"gt\": [100]} matches records where field \u003e 100", - "type": "array", - "items": {} - }, - "gte": { - "description": "GreaterOrEqual specifies values that must be greater than or equal to the provided value.\nShould contain exactly one value for comparison.\nExample: {\"gte\": [\"2025-06-01\"]} matches records where field \u003e= \"2025-06-01\"", - "type": "array", - "items": {} - }, - "in": { - "description": "In specifies a list of values where the field must match any one of them.\nMultiple values treated as OR conditions.\nExample: {\"in\": [\"active\", \"pending\", \"suspended\"]} matches any of these statuses", - "type": "array", - "items": {} - }, - "like": { - "description": "Like specifies values that must match the provided value using LIKE pattern matching.\nShould contain exactly one value for comparison.\nExample: {\"like\": [\"%active%\"]} matches any status containing \"active\"", - "type": "array", - "items": {} - }, - "lt": { - "description": "LessThan specifies values that must be less than the provided value.\nShould contain exactly one value for comparison.\nExample: {\"lt\": [1000]} matches records where field \u003c 1000", - "type": "array", - "items": {} - }, - "lte": { - "description": "LessOrEqual specifies values that must be less than or equal to the provided value.\nShould contain exactly one value for comparison.\nExample: {\"lte\": [\"2025-06-30\"]} matches records where field \u003c= \"2025-06-30\"", - "type": "array", - "items": {} - }, - "ne": { - "description": "NotEquals specifies values that must NOT match the provided value.\nShould contain exactly one value for comparison.\nExample: {\"ne\": [\"active\"]} excludes this status", - "type": "array", - "items": {} - }, - "nin": { - "description": "NotIn specifies a list of values where the field must NOT match any of them.\nMultiple values treated as AND NOT conditions.\nExample: {\"nin\": [\"deleted\", \"archived\"]} excludes these statuses", - "type": "array", - "items": {} - } - } - }, - "pkg.HTTPError": { - "type": "object", - "properties": { - "code": { - "type": "string" - }, - "entityType": { - "type": "string" - }, - "err": {}, - "message": { - "type": "string" - }, - "title": { - "type": "string" - } - } - } - } -}` - -// SwaggerInfo holds exported Swagger Info so clients can modify it -var SwaggerInfo = &swag.Spec{ - Version: "1.0.0", - Host: "localhost:4006", - BasePath: "/", - Schemes: []string{}, - Title: "Fetcher Manager API", - Description: "API documentation for the Fetcher Manager component", - InfoInstanceName: "swagger", - SwaggerTemplate: docTemplate, - LeftDelim: "{{", - RightDelim: "}}", -} - -func init() { - swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) -} diff --git a/components/manager/api/openapi.yaml b/components/manager/api/openapi.yaml index b215dcd0..80be2a46 100644 --- a/components/manager/api/openapi.yaml +++ b/components/manager/api/openapi.yaml @@ -1,1322 +1,1673 @@ -openapi: 3.0.1 -info: - contact: {} - description: API documentation for the Fetcher Manager component - termsOfService: http://swagger.io/terms/ - title: Fetcher Manager API - version: 1.0.0 -servers: -- url: //localhost:4006/ -paths: - /v1/fetcher: - post: - description: Create a new data extraction job. The request will be validated, - deduplicated within a 5-minute window, and all referenced connections will - be tested before job creation. The metadata.source field is required for product - isolation and datasource ownership validation. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: +components: + schemas: + ConnectionInput: + additionalProperties: false + properties: + configName: + description: Unique configuration name used to reference this connection. + examples: + - production-db type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.FetcherRequest' - description: Fetcher request payload. metadata.source is required. - required: true - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse' - description: Duplicate request - returning existing job - "202": - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse' - description: Job created and queued for processing - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Bad Request - "409": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Conflict - "413": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Request Entity Too Large - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Internal Server Error - summary: Create fetcher job - tags: - - Fetcher - x-codegen-request-body-name: request - /v1/fetcher/{id}: - get: - description: Retrieve detailed information about a specific data extraction - job. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: + databaseName: + description: Database name to connect to. + examples: + - mydatabase type: string - - description: Job ID - in: path - name: id - required: true - schema: + host: + description: Database server hostname or IP address. + examples: + - db.example.com type: string - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.JobResponse' - description: OK - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Bad Request - "404": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Not Found - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Internal Server Error - summary: Get fetcher job - tags: - - Fetcher - /v1/management/connections: - get: - description: List connections with pagination and filters. When X-Product-Name - is provided, returns only connections associated with that product. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: + metadata: + additionalProperties: {} + description: Optional application-defined connection metadata. + examples: + - environment: production + region: us-east-1 + type: object + password: + description: Database authentication password. + examples: + - secretpassword type: string - - description: Product name. When provided, filters connections by product. - in: header - name: X-Product-Name + port: + description: Database server TCP port. + examples: + - 5432 + format: int64 + type: integer schema: + description: Optional default database schema. + examples: + - public type: string - - description: Page number (minimum 1) - in: query - name: page - schema: - default: 1 + ssl: + $ref: "#/components/schemas/SSLInput" + description: Optional SSL/TLS connection settings. + examples: + - ca: certificate-authority + mode: require + type: + description: Database engine type. + examples: + - POSTGRESQL + type: string + userName: + description: Database authentication username. + examples: + - dbuser + type: string + required: + - configName + - type + - host + - port + - databaseName + - userName + - password + type: object + ConnectionPage: + additionalProperties: false + properties: + items: + description: Connections returned for this page. + examples: + - - configName: production-db + createdAt: "2026-01-15T10:30:00Z" + databaseName: mydatabase + host: db.example.com + id: 018f47a6-3e5f-7b9a-8c1d-2e3f4a5b6c7d + metadata: + environment: production + region: us-east-1 + port: 5432 + productName: midaz + schema: public + ssl: + mode: require + type: POSTGRESQL + updatedAt: "2026-01-16T14:45:00Z" + userName: dbuser + items: + $ref: "#/components/schemas/ConnectionResponse" + type: + - array + - "null" + limit: + description: Maximum items returned per page. + examples: + - 10 + format: int64 type: integer - - description: Page size (default 50, max 1000) - in: query - name: limit - schema: - default: 50 + page: + description: Current page number. + examples: + - 1 + format: int64 type: integer - - description: Sort order - in: query - name: sortOrder - schema: - default: desc - enum: - - asc - - desc - type: string - - description: Filter by database type - in: query - name: type - schema: - enum: - - ORACLE - - SQL_SERVER - - POSTGRESQL - - MONGODB - - MYSQL - type: string - - description: Filter by host - in: query - name: host - schema: + total: + description: Total matching connections. + examples: + - 10 + format: int64 + type: integer + required: + - items + - limit + - total + type: object + ConnectionResponse: + additionalProperties: false + properties: + configName: + description: Configuration name used to reference the connection. + examples: + - production-db type: string - - description: Filter by database name - in: query - name: databaseName - schema: + createdAt: + description: Timestamp when the connection was created. + examples: + - "2026-01-15T10:30:00Z" + format: date-time type: string - - description: Filter by start date (YYYY-MM-DD) - in: query - name: startDate - schema: + databaseName: + description: Connected database name. + examples: + - mydatabase type: string - - description: Filter by end date (YYYY-MM-DD) - in: query - name: endDate - schema: + host: + description: Database server hostname or IP address. + examples: + - db.example.com type: string - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200' - description: OK - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Bad Request - "404": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Not Found - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Internal Server Error - summary: List connections - tags: - - Connections - post: - description: Create a new database connection for the organization. The X-Product-Name - header is required and identifies the product for this connection. ConfigName - must be unique per organization (and per product when assigned); password - is encrypted and a UUID is generated on creation. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: + id: + description: Unique connection identifier. + examples: + - 018f47a6-3e5f-7b9a-8c1d-2e3f4a5b6c7d type: string - - description: Product name (required, non-empty) - in: header - name: X-Product-Name - required: true - schema: + metadata: + additionalProperties: {} + description: Application-defined connection metadata. + examples: + - environment: production + region: us-east-1 + type: object + port: + description: Database server TCP port. + examples: + - 5432 + format: int64 + type: integer + productName: + description: Product that owns the connection. + examples: + - midaz type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionInput' - description: Connection payload - required: true - responses: - "201": - content: - application/json: - schema: - additionalProperties: - type: string - type: object - description: Created connection identifier - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Bad Request - "409": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Conflict - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Internal Server Error - summary: Create connection - tags: - - Connections - x-codegen-request-body-name: connection - /v1/management/connections/unassigned: - get: - description: List connections that have no product assigned, useful for migration - purposes. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization schema: + description: Default database schema, when configured. + examples: + - public type: string - - description: Page number (minimum 1) - in: query - name: page - schema: - default: 1 + ssl: + $ref: "#/components/schemas/SSLResponse" + description: Configured SSL/TLS settings with secrets omitted. + examples: + - mode: require + type: + description: Database engine type. + examples: + - POSTGRESQL + type: string + updatedAt: + description: Timestamp when the connection was last updated. + examples: + - "2026-01-16T14:45:00Z" + format: date-time + type: string + userName: + description: Database authentication username. + examples: + - dbuser + type: string + required: + - id + - configName + - type + - host + - port + - databaseName + - userName + - createdAt + - updatedAt + type: object + ConnectionSchemaResponse: + additionalProperties: false + properties: + configName: + description: Configuration name used to reference the connection. + examples: + - production-db + type: string + databaseName: + description: Database whose schema was discovered. + examples: + - mydatabase + type: string + id: + description: Unique connection identifier. + examples: + - 018f47a6-3e5f-7b9a-8c1d-2e3f4a5b6c7d + type: string + tables: + description: Discovered tables or collections and their fields. + examples: + - - fields: + - id + - name + - created_at + name: public.accounts + items: + $ref: "#/components/schemas/TableDetails" + type: + - array + - "null" + type: + description: Database engine type. + examples: + - POSTGRESQL + type: string + required: + - id + - configName + - databaseName + - type + - tables + type: object + ConnectionTestResponse: + additionalProperties: false + properties: + latencyMs: + description: Round-trip connection latency in milliseconds. + examples: + - 18 + format: int64 type: integer - - description: Page size (default 50, max 1000) - in: query - name: limit - schema: - default: 50 + message: + description: Human-readable connectivity test result. + examples: + - Connection established successfully + type: string + status: + description: Connectivity test outcome. + examples: + - success + type: string + required: + - status + - message + - latencyMs + type: object + ConnectionUpdateInput: + additionalProperties: false + properties: + configName: + description: New configuration name for the connection. + examples: + - production-db + type: string + databaseName: + description: New database name to connect to. + examples: + - mydatabase + type: string + host: + description: New database server hostname or IP address. + examples: + - db.example.com + type: string + metadata: + additionalProperties: {} + description: Application-defined metadata that replaces the current metadata. + examples: + - environment: production + region: us-east-1 + type: object + password: + description: New database authentication password. + examples: + - secretpassword + type: string + port: + description: New database server TCP port. + examples: + - 5432 + format: int64 type: integer - - description: Sort order - in: query - name: sortOrder schema: - default: desc - enum: - - asc - - desc + description: New default database schema. + examples: + - public type: string - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200' - description: OK - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Bad Request - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Internal Server Error - summary: List unassigned connections - tags: - - Migration - /v1/management/connections/validate-schema: - post: - description: Validate that tables and fields referenced in the request exist - in the configured datasources. Returns 200 when validation passes, 422 when - validation fails with detailed error information. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: + ssl: + $ref: "#/components/schemas/SSLUpdateInput" + description: SSL/TLS settings to update. + examples: + - mode: verify-full + type: + description: New database engine type. + examples: + - POSTGRESQL type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest' - description: Schema validation request - required: true - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse' - description: Validation successful - all tables and fields exist - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Invalid request payload or missing headers - "422": + userName: + description: New database authentication username. + examples: + - dbuser + type: string + type: object + DataRequest: + additionalProperties: false + properties: + filters: + additionalProperties: + additionalProperties: + additionalProperties: + $ref: "#/components/schemas/FilterCondition" + type: object + type: object + description: Optional filter conditions grouped by data source, table, and field. + examples: + - ledger: + transactions: + status: + eq: + - approved + type: object + mappedFields: + additionalProperties: + additionalProperties: + items: + type: string + type: + - array + - "null" + type: object + description: Fields to extract, grouped by data source and table. + examples: + - ledger: + transactions: + - id + - amount + type: object + required: + - mappedFields + type: object + Detail: + additionalProperties: false + properties: + code: + description: "Stable, machine-readable domain error code scoped to the emitting service (format: -NNNN)." + examples: + - ERR-0001 + type: string + detail: + description: A human-readable explanation specific to this occurrence of the problem. + examples: + - Property foo is required but is missing. + type: string + errors: + description: Optional list of individual error details + items: + $ref: "#/components/schemas/ErrorDetail" + type: + - array + - "null" + instance: + description: A URI reference that identifies the specific occurrence of the problem. + examples: + - https://example.com/error-log/abc123 + format: uri + type: string + status: + description: HTTP status code + examples: + - 400 + format: int64 + type: integer + title: + description: A short, human-readable summary of the problem type. This value should not change between occurrences of the error. + examples: + - Bad Request + type: string + type: + default: about:blank + description: A URI reference to human-readable documentation for the error. + examples: + - https://example.com/errors/example + format: uri + type: string + type: object + ErrorDetail: + additionalProperties: false + properties: + location: + description: Where the error occurred, e.g. 'body.items[3].tags' or 'path.thing-id' + type: string + message: + description: Error message text + type: string + value: + description: The value at the given location + type: object + FetcherRequest: + additionalProperties: false + properties: + dataRequest: + $ref: "#/components/schemas/DataRequest" + description: Fields and filters to extract from configured data sources. + examples: + - mappedFields: + ledger: + transactions: + - id + - amount + metadata: + additionalProperties: {} + description: Caller-defined metadata stored with the job; source identifies the owning product. + examples: + - correlationId: settlement-2026-07-13 + source: payments + properties: + source: + description: Owning product used for notification routing and connection ownership validation. + examples: + - payments + type: string + required: + - source + type: object + required: + - dataRequest + - metadata + type: object + FetcherResponse: + additionalProperties: false + properties: + createdAt: + description: UTC timestamp when the job was created. + examples: + - "2026-07-13T12:00:00Z" + format: date-time + type: string + jobId: + description: Unique identifier of the extraction job. + examples: + - 01980a89-21f0-7d7e-a109-564b5c6f53ac + type: string + message: + description: Outcome of the create request. + examples: + - Job created and queued for processing + type: string + status: + description: Current processing status of the job. + examples: + - pending + type: string + required: + - jobId + - status + - createdAt + - message + type: object + FilterCondition: + additionalProperties: false + properties: + between: + description: Inclusive lower and upper bounds for the field. + examples: + - - 100 + - 1000 + items: {} + type: + - array + - "null" + eq: + description: Values that the field may equal. + examples: + - - active + - pending + items: {} + type: + - array + - "null" + gt: + description: Single value that the field must be greater than. + examples: + - - 100 + items: {} + type: + - array + - "null" + gte: + description: Single value that the field must be greater than or equal to. + examples: + - - "2025-06-01" + items: {} + type: + - array + - "null" + in: + description: Values that the field may match. + examples: + - - active + - pending + - suspended + items: {} + type: + - array + - "null" + like: + description: Single LIKE pattern that the field must match. + examples: + - - "%active%" + items: {} + type: + - array + - "null" + lt: + description: Single value that the field must be less than. + examples: + - - 1000 + items: {} + type: + - array + - "null" + lte: + description: Single value that the field must be less than or equal to. + examples: + - - "2025-06-30" + items: {} + type: + - array + - "null" + ne: + description: Single value that the field must not equal. + examples: + - - active + items: {} + type: + - array + - "null" + nin: + description: Values that the field must not match. + examples: + - - deleted + - archived + items: {} + type: + - array + - "null" + type: object + JobResponse: + additionalProperties: false + properties: + completedAt: + description: UTC timestamp when the job reached a terminal state. + examples: + - "2026-07-13T12:05:00Z" + format: date-time + type: string + createdAt: + description: UTC timestamp when the job was created. + examples: + - "2026-07-13T12:00:00Z" + format: date-time + type: string + filters: + additionalProperties: + additionalProperties: + additionalProperties: + $ref: "#/components/schemas/FilterCondition" + type: object + type: object + description: Filter conditions applied to the extraction. + examples: + - ledger: + transactions: + status: + eq: + - approved + type: object + id: + description: Unique identifier of the extraction job. + examples: + - 01980a89-21f0-7d7e-a109-564b5c6f53ac + type: string + mappedFields: + additionalProperties: + additionalProperties: + items: + type: string + type: + - array + - "null" + type: object + description: Fields requested for extraction, grouped by data source and table. + examples: + - ledger: + transactions: + - id + - amount + type: object + metadata: + additionalProperties: {} + description: Caller-defined public metadata stored with the job. + examples: + - correlationId: settlement-2026-07-13 + source: payments + type: object + requestHash: + description: SHA-256 digest used to detect idempotent duplicate requests. + examples: + - d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592 + type: string + resultHmac: + description: HMAC digest used to verify the result integrity. + examples: + - 9dc118e1d4efdcf0f45bb646c38c2a02e9c20b0fc2c75c1a9be0fae200bd9f93 + type: string + resultPath: + description: Storage path of the generated extraction result. + examples: + - s3://fetcher-results/jobs/01980a89-21f0-7d7e-a109-564b5c6f53ac.json + type: string + status: + description: Current processing status of the job. + examples: + - completed + type: string + required: + - id + - mappedFields + - status + - createdAt + type: object + SSLInput: + additionalProperties: false + properties: + ca: + description: PEM-encoded certificate authority used to verify the server. + examples: + - |- + -----BEGIN CERTIFICATE----- + ... + type: string + cert: + description: Optional PEM-encoded client certificate. + examples: + - |- + -----BEGIN CERTIFICATE----- + ... + type: string + key: + description: Optional PEM-encoded private key for the client certificate. + examples: + - |- + -----BEGIN PRIVATE KEY----- + ... + type: string + mode: + description: Driver-specific SSL/TLS mode. + examples: + - require + type: string + required: + - mode + - ca + type: object + SSLResponse: + additionalProperties: false + properties: + mode: + description: Configured driver-specific SSL/TLS mode. + examples: + - require + type: string + type: object + SSLUpdateInput: + additionalProperties: false + properties: + ca: + description: New PEM-encoded certificate authority used to verify the server. + examples: + - |- + -----BEGIN CERTIFICATE----- + ... + type: string + cert: + description: New PEM-encoded client certificate. + examples: + - |- + -----BEGIN CERTIFICATE----- + ... + type: string + key: + description: New PEM-encoded private key for the client certificate. + examples: + - |- + -----BEGIN PRIVATE KEY----- + ... + type: string + mode: + description: New driver-specific SSL/TLS mode. + examples: + - require + type: string + type: object + SchemaValidationError: + additionalProperties: false + properties: + dataSourceId: + description: Datasource configuration name where validation failed. + examples: + - midaz_onboarding + type: string + field: + description: Field that could not be found, when applicable. + examples: + - external_id + type: string + table: + description: Table or collection where validation failed. + examples: + - accounts + type: string + type: + description: Machine-readable validation failure type. + examples: + - FIELD_NOT_FOUND + type: string + required: + - type + - dataSourceId + type: object + SchemaValidationRequest: + additionalProperties: false + properties: + mappedFields: + additionalProperties: + additionalProperties: + items: + type: string + type: + - array + - "null" + type: object + description: Maps each datasource configuration name to tables and the fields required from each table. + examples: + - midaz_onboarding: + accounts: + - id + - name + type: object + required: + - mappedFields + type: object + SchemaValidationResponse: + additionalProperties: false + properties: + errors: + description: Validation failures, omitted when every requested field is valid. + examples: + - - dataSourceId: midaz_onboarding + field: external_id + table: accounts + type: FIELD_NOT_FOUND + items: + $ref: "#/components/schemas/SchemaValidationError" + type: + - array + - "null" + message: + description: Human-readable schema validation result. + examples: + - Schema validation completed successfully + type: string + status: + description: Schema validation outcome. + examples: + - success + type: string + required: + - status + - message + type: object + TableDetails: + additionalProperties: false + properties: + fields: + description: Fields discovered in the table or collection. + examples: + - - id + - name + - created_at + items: + type: string + type: + - array + - "null" + name: + description: Qualified table or collection name. + examples: + - public.accounts + type: string + required: + - name + - fields + type: object + securitySchemes: + BearerAuth: + bearerFormat: JWT + description: JWT bearer token issued by the identity provider. + scheme: bearer + type: http +info: + contact: + email: contact@lerian.studio + name: Lerian + url: https://github.com/LerianStudio/fetcher/discussions + description: API documentation for the Fetcher Manager component + license: + identifier: Elastic-2.0 + name: Elastic License 2.0 (Source Available) + title: Fetcher Manager API + version: 1.0.0 +openapi: 3.1.0 +paths: + /v1/fetcher: + post: + description: Creates and queues a data extraction job, or returns the existing job for an idempotent duplicate request. + operationId: create-fetcher-job + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FetcherRequest" + description: Fetcher request payload. + required: true + responses: + "200": content: application/json: schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse' - description: Validation failed - schema errors found (missing tables, fields, - or unreachable datasources) - "500": + $ref: "#/components/schemas/FetcherResponse" + description: Duplicate request; the existing job is returned. + "202": content: application/json: schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Internal server error - summary: Validate schema - tags: - - Connections - x-codegen-request-body-name: request - /v1/management/connections/{id}: - delete: - description: Soft delete a connection when no active jobs are running for it. - Returns 409 if there is any active job. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: - type: string - - description: Connection ID - in: path - name: id - required: true - schema: - type: string - responses: - "204": - content: {} - description: No Content + $ref: "#/components/schemas/FetcherResponse" + description: Accepted "400": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Bad Request - "404": + "401": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Not Found + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden "409": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Conflict + "413": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Request Entity Too Large + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity "500": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Internal Server Error - summary: Delete connection + summary: Create a data extraction job tags: - - Connections + - Fetcher + /v1/fetcher/{id}: get: - description: Get connection details by ID for the given organization. + description: Returns a data extraction job by identifier. + operationId: get-fetcher-job parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: - type: string - - description: Connection ID - in: path - name: id - required: true - schema: - type: string + - description: Resource identifier. + in: path + name: id + required: true + schema: + description: Resource identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string responses: "200": content: application/json: schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' + $ref: "#/components/schemas/JobResponse" description: OK "400": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden "404": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Not Found + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity "500": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Internal Server Error - summary: Get connection + summary: Get a data extraction job tags: - - Connections - patch: - description: Apply a partial update to a connection. Only include fields you - want to change. Returns 409 if there is any active job. + - Fetcher + /v1/management/connections: + get: + description: Returns a filtered, paginated list of database connections visible to the optional product. Dynamic metadata filters use query parameters such as metadata.region=br. + operationId: list-connections parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: - type: string - - description: Connection ID - in: path - name: id - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput' - description: Fields to update (only include fields you want to change) - required: true + - description: Product name used for connection isolation. + in: header + name: X-Product-Name + schema: + examples: + - midaz + type: string + - description: Page number; defaults to 1. + in: query + name: page + schema: + default: 1 + examples: + - 1 + type: integer + - description: Page size; defaults to 10 and is capped at 100. + in: query + name: limit + schema: + default: 10 + examples: + - 10 + type: integer + - description: Sort direction. + in: query + name: sortOrder + schema: + default: desc + enum: + - asc + - desc + examples: + - desc + type: string + - description: Database type filter. + in: query + name: type + schema: + enum: + - ORACLE + - SQL_SERVER + - POSTGRESQL + - MONGODB + - MYSQL + examples: + - POSTGRESQL + type: string + - description: Inclusive creation date in YYYY-MM-DD format. + in: query + name: startDate + schema: + examples: + - "2026-01-01" + type: string + - description: Inclusive final creation date in YYYY-MM-DD format. + in: query + name: endDate + schema: + examples: + - "2026-01-31" + type: string responses: "200": content: application/json: schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' + $ref: "#/components/schemas/ConnectionPage" description: OK "400": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Bad Request - "404": + "401": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Not Found - "409": + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' - description: Conflict + $ref: "#/components/schemas/Detail" + description: Forbidden "500": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Internal Server Error - summary: Update connection + summary: List connections tags: - - Connections - x-codegen-request-body-name: connection - /v1/management/connections/{id}/assign: + - Connections post: - description: Associate an unassigned connection to a product. This is a one-time, - irreversible operation for migration purposes. The product name must be provided - via the X-Product-Name header. + description: Creates a database connection for the product named by the X-Product-Name header. + operationId: create-connection parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: - type: string - - description: Connection ID - in: path - name: id + - description: Product name used for connection isolation. + in: header + name: X-Product-Name + required: true + schema: + examples: + - midaz + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionInput" + description: Connection settings and credentials. required: true - schema: - type: string - - description: Product name to assign - in: header - name: X-Product-Name + responses: + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionResponse" + description: Created + "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden + "409": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Conflict + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity + "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Internal Server Error + summary: Create connection + tags: + - Connections + /v1/management/connections/unassigned: + get: + description: List connections that have no product assigned, useful for migration purposes. + operationId: list-unassigned-connections + parameters: + - description: Page number; defaults to 1. + in: query + name: page + schema: + default: 1 + examples: + - 1 + type: integer + - description: Page size; defaults to 10 and is capped at 100. + in: query + name: limit + schema: + default: 10 + examples: + - 10 + type: integer + - description: Sort direction. + in: query + name: sortOrder + schema: + default: desc + enum: + - asc + - desc + examples: + - desc + type: string + - description: Inclusive creation date in YYYY-MM-DD format. + in: query + name: startDate + schema: + examples: + - "2026-01-01" + type: string + - description: Inclusive final creation date in YYYY-MM-DD format. + in: query + name: endDate + schema: + examples: + - "2026-01-31" + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionPage" + description: OK + "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden + "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Internal Server Error + summary: List unassigned connections + tags: + - Migration + /v1/management/connections/validate-schema: + post: + description: Validates mapped datasource, table, and field identifiers before a connection is used. + operationId: validate-schema + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SchemaValidationRequest" + description: Datasource, table, and field mappings to validate. required: true - schema: - type: string responses: "200": content: application/json: schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' + $ref: "#/components/schemas/SchemaValidationResponse" description: OK "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity + "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Internal Server Error + summary: Validate a connection schema + tags: + - Connections + /v1/management/connections/{id}: + delete: + description: Deletes a connection when it has no active jobs. + operationId: delete-connection + parameters: + - description: Resource identifier. + in: path + name: id + required: true + schema: + description: Resource identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string + responses: + "204": + description: No Content + "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden + "404": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Not Found + "409": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Conflict + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity + "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Internal Server Error + summary: Delete a connection + tags: + - Connections + get: + description: Returns one database connection by identifier. + operationId: get-connection + parameters: + - description: Resource identifier. + in: path + name: id + required: true + schema: + description: Resource identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string + responses: + "200": content: application/json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/ConnectionResponse" + description: OK + "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden "404": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Not Found + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity + "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Internal Server Error + summary: Get a connection + tags: + - Connections + patch: + description: Applies a partial update to an existing database connection. + operationId: update-connection + parameters: + - description: Connection identifier. + in: path + name: id + required: true + schema: + description: Connection identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionUpdateInput" + description: Connection fields to update; omitted fields remain unchanged. + required: true + responses: + "200": content: application/json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/ConnectionResponse" + description: OK + "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden + "404": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" description: Not Found "409": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Conflict + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Internal Server Error + summary: Update a connection + tags: + - Connections + /v1/management/connections/{id}/assign: + post: + description: Associate an unassigned connection to a product. This is a one-time, irreversible operation for migration purposes. The product name must be provided via the X-Product-Name header. + operationId: assign-connection-to-product + parameters: + - description: Product name used for connection isolation. + in: header + name: X-Product-Name + required: true + schema: + examples: + - midaz + type: string + - description: Resource identifier. + in: path + name: id + required: true + schema: + description: Resource identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string + responses: + "200": content: application/json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/ConnectionResponse" + description: OK + "400": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden + "404": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Not Found + "409": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Conflict + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity + "500": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" description: Internal Server Error summary: Assign connection to product tags: - - Migration + - Migration /v1/management/connections/{id}/schema: get: - description: Get the database schema (tables and fields) for a connection. + description: Returns tables and fields discovered from the configured datasource. + operationId: get-connection-schema parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: - type: string - - description: Connection ID - in: path - name: id - required: true - schema: - type: string + - description: Resource identifier. + in: path + name: id + required: true + schema: + description: Resource identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string responses: "200": content: application/json: schema: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse' + $ref: "#/components/schemas/ConnectionSchemaResponse" description: OK "400": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden "404": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Not Found + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity "500": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Internal Server Error - summary: Get connection schema + summary: Discover a connection schema tags: - - Connections + - Connections /v1/management/connections/{id}/test: post: - description: Test the configured connection by establishing and closing a connection. + description: Attempts to connect to the configured datasource and reports the observed latency. + operationId: test-connection parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - schema: - type: string - - description: Connection ID - in: path - name: id - required: true - schema: - type: string + - description: Resource identifier. + in: path + name: id + required: true + schema: + description: Resource identifier. + examples: + - 2f1f35f4-4f50-44f3-a8ab-2772d395f0c2 + type: string responses: "200": content: application/json: schema: - additionalProperties: - type: object - type: object - description: Connection test result + $ref: "#/components/schemas/ConnectionTestResponse" + description: OK "400": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Bad Request + "401": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unauthorized + "403": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Forbidden "404": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Not Found + "422": + content: + application/problem+json: + schema: + $ref: "#/components/schemas/Detail" + description: Unprocessable Entity "429": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Too Many Requests "500": content: - application/json: + application/problem+json: schema: - $ref: '#/components/schemas/pkg.HTTPError' + $ref: "#/components/schemas/Detail" description: Internal Server Error - summary: Test connection + summary: Test a connection tags: - - Connections -components: - schemas: - Pagination: - description: Pagination is the struct designed to store the pagination data - of an entity list. - example: - total: 10 - limit: 10 - page: 1 - items: '{}' - properties: - items: - type: object - limit: - example: 10 - type: integer - page: - example: 1 - type: integer - total: - example: 10 - type: integer - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionInput: - properties: - configName: - example: production-db - maxLength: 100 - minLength: 3 - type: string - databaseName: - example: mydatabase - type: string - host: - example: db.example.com - type: string - metadata: - additionalProperties: - type: object - type: object - password: - example: secretpassword - type: string - port: - example: 5432 - maximum: 65535 - minimum: 1 - type: integer - schema: - example: my_schema - type: string - ssl: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SSLInput' - type: - enum: - - ORACLE - - SQL_SERVER - - POSTGRESQL - - MONGODB - - MYSQL - example: POSTGRESQL - type: string - userName: - example: dbuser - type: string - required: - - configName - - databaseName - - host - - password - - port - - type - - userName - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse: - example: - configName: configName - schema: schema - metadata: - key: '{}' - databaseName: databaseName - type: type - userName: userName - ssl: - mode: mode - productName: productName - createdAt: createdAt - port: 0 - host: host - id: id - updatedAt: updatedAt - properties: - configName: - type: string - createdAt: - type: string - databaseName: - type: string - host: - type: string - id: - type: string - metadata: - additionalProperties: - type: object - type: object - port: - type: integer - productName: - type: string - schema: - type: string - ssl: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SSLResponse' - type: - type: string - updatedAt: - type: string - userName: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse: - example: - configName: configName - tables: - - name: name - fields: - - fields - - fields - - name: name - fields: - - fields - - fields - databaseName: databaseName - id: id - type: type - properties: - configName: - type: string - databaseName: - type: string - id: - type: string - tables: - items: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.TableDetails' - type: array - type: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput: - properties: - configName: - example: production-db - maxLength: 100 - minLength: 3 - type: string - databaseName: - example: mydatabase - type: string - host: - example: db.example.com - type: string - metadata: - additionalProperties: - type: object - type: object - password: - example: secretpassword - type: string - port: - example: 5432 - maximum: 65535 - minimum: 1 - type: integer - schema: - example: my_schema - type: string - ssl: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput' - type: - enum: - - ORACLE - - SQL_SERVER - - POSTGRESQL - - MONGODB - - MYSQL - example: POSTGRESQL - type: string - userName: - example: dbuser - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.DataRequest: - description: DataRequest encapsulates field mappings and optional filters for - data extraction. - properties: - filters: - additionalProperties: - additionalProperties: - additionalProperties: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition' - type: object - type: object - type: object - mappedFields: - additionalProperties: - additionalProperties: - items: - type: string - type: array - type: object - type: object - required: - - mappedFields - type: object - github_com_LerianStudio_fetcher_pkg_model.FetcherRequest: - description: FetcherRequest represents the request body for creating a new data - extraction job. - properties: - dataRequest: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.DataRequest' - metadata: - additionalProperties: - type: object - type: object - required: - - dataRequest - type: object - github_com_LerianStudio_fetcher_pkg_model.FetcherResponse: - description: FetcherResponse represents the response after successfully creating - a data extraction job. - example: - createdAt: createdAt - jobId: jobId - message: message - status: status - properties: - createdAt: - type: string - jobId: - type: string - message: - type: string - status: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.JobResponse: - description: JobResponse represents the complete information about a data extraction - job. - example: - resultHmac: resultHmac - createdAt: createdAt - mappedFields: - key: - key: - - mappedFields - - mappedFields - completedAt: completedAt - metadata: - key: '{}' - requestHash: requestHash - resultPath: resultPath - filters: - key: - key: - key: - nin: - - '{}' - - '{}' - in: - - '{}' - - '{}' - like: - - '{}' - - '{}' - ne: - - '{}' - - '{}' - lt: - - '{}' - - '{}' - gte: - - '{}' - - '{}' - eq: - - '{}' - - '{}' - lte: - - '{}' - - '{}' - gt: - - '{}' - - '{}' - between: - - '{}' - - '{}' - id: id - status: status - properties: - completedAt: - type: string - createdAt: - type: string - filters: - additionalProperties: - additionalProperties: - additionalProperties: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition' - type: object - type: object - type: object - id: - type: string - mappedFields: - additionalProperties: - additionalProperties: - items: - type: string - type: array - type: object - type: object - metadata: - additionalProperties: - type: object - type: object - requestHash: - type: string - resultHmac: - type: string - resultPath: - type: string - status: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.NestedFilters: - additionalProperties: - additionalProperties: - additionalProperties: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition' - type: object - type: object - type: object - github_com_LerianStudio_fetcher_pkg_model.SSLInput: - properties: - ca: - example: |- - -----BEGIN CERTIFICATE----- - ... - type: string - cert: - type: string - key: - type: string - mode: - example: require - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SSLResponse: - example: - mode: mode - properties: - mode: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput: - properties: - ca: - example: |- - -----BEGIN CERTIFICATE----- - ... - type: string - cert: - type: string - key: - type: string - mode: - example: require - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError: - example: - dataSourceId: dataSourceId - field: field - type: type - table: table - properties: - dataSourceId: - type: string - field: - type: string - table: - type: string - type: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse: - properties: - code: - type: string - errors: - items: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError' - type: array - message: - type: string - title: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest: - description: Request body for schema validation containing mapped fields per - datasource. - properties: - mappedFields: - additionalProperties: - additionalProperties: - items: - type: string - type: array - type: object - description: |- - MappedFields maps datasource config names to their tables and fields - Key: configName (e.g., "midaz_onboarding") - Value: map of table names to field names - type: object - required: - - mappedFields - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse: - example: - message: message - errors: - - dataSourceId: dataSourceId - field: field - type: type - table: table - - dataSourceId: dataSourceId - field: field - type: type - table: table - status: status - properties: - errors: - items: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError' - type: array - message: - type: string - status: - description: '"success" or "failure"' - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.TableDetails: - example: - name: name - fields: - - fields - - fields - properties: - fields: - items: - type: string - type: array - name: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition: - example: - nin: - - '{}' - - '{}' - in: - - '{}' - - '{}' - like: - - '{}' - - '{}' - ne: - - '{}' - - '{}' - lt: - - '{}' - - '{}' - gte: - - '{}' - - '{}' - eq: - - '{}' - - '{}' - lte: - - '{}' - - '{}' - gt: - - '{}' - - '{}' - between: - - '{}' - - '{}' - properties: - between: - description: |- - Between specifies a range condition with exactly two values [min, max]. - Matches records where min <= field <= max - Example: {"between": [100, 1000]} matches records where 100 <= field <= 1000 - items: - type: object - type: array - eq: - description: |- - Equals specifies exact value matches. Multiple values treated as OR conditions. - Example: {"eq": ["active", "pending"]} matches records where field equals "active" OR "pending" - items: - type: object - type: array - gt: - description: |- - GreaterThan specifies values that must be greater than the provided value. - Should contain exactly one value for comparison. - Example: {"gt": [100]} matches records where field > 100 - items: - type: object - type: array - gte: - description: |- - GreaterOrEqual specifies values that must be greater than or equal to the provided value. - Should contain exactly one value for comparison. - Example: {"gte": ["2025-06-01"]} matches records where field >= "2025-06-01" - items: - type: object - type: array - in: - description: |- - In specifies a list of values where the field must match any one of them. - Multiple values treated as OR conditions. - Example: {"in": ["active", "pending", "suspended"]} matches any of these statuses - items: - type: object - type: array - like: - description: |- - Like specifies values that must match the provided value using LIKE pattern matching. - Should contain exactly one value for comparison. - Example: {"like": ["%active%"]} matches any status containing "active" - items: - type: object - type: array - lt: - description: |- - LessThan specifies values that must be less than the provided value. - Should contain exactly one value for comparison. - Example: {"lt": [1000]} matches records where field < 1000 - items: - type: object - type: array - lte: - description: |- - LessOrEqual specifies values that must be less than or equal to the provided value. - Should contain exactly one value for comparison. - Example: {"lte": ["2025-06-30"]} matches records where field <= "2025-06-30" - items: - type: object - type: array - ne: - description: |- - NotEquals specifies values that must NOT match the provided value. - Should contain exactly one value for comparison. - Example: {"ne": ["active"]} excludes this status - items: - type: object - type: array - nin: - description: |- - NotIn specifies a list of values where the field must NOT match any of them. - Multiple values treated as AND NOT conditions. - Example: {"nin": ["deleted", "archived"]} excludes these statuses - items: - type: object - type: array - type: object - pkg.HTTPError: - properties: - code: - type: string - entityType: - type: string - err: - type: object - message: - type: string - title: - type: string - type: object - inline_response_200: - example: - Pagination: - total: 10 - limit: 10 - page: 1 - items: '{}' - total: 5 - limit: 6 - page: 1 - items: - - configName: configName - schema: schema - metadata: - key: '{}' - databaseName: databaseName - type: type - userName: userName - ssl: - mode: mode - productName: productName - createdAt: createdAt - port: 0 - host: host - id: id - updatedAt: updatedAt - - configName: configName - schema: schema - metadata: - key: '{}' - databaseName: databaseName - type: type - userName: userName - ssl: - mode: mode - productName: productName - createdAt: createdAt - port: 0 - host: host - id: id - updatedAt: updatedAt - properties: - Pagination: - $ref: '#/components/schemas/Pagination' - items: - items: - $ref: '#/components/schemas/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' - type: array - limit: - type: integer - page: - type: integer - total: - type: integer - type: object -x-original-swagger-version: "2.0" + - Connections +security: + - BearerAuth: [] +tags: + - description: Database connection lifecycle and schema operations. + name: Connections + - description: Legacy connection assignment and migration operations. + name: Migration + - description: Data extraction job creation and status operations. + name: Fetcher diff --git a/components/manager/api/swagger.json b/components/manager/api/swagger.json deleted file mode 100644 index 3c3c8d2c..00000000 --- a/components/manager/api/swagger.json +++ /dev/null @@ -1,1377 +0,0 @@ -{ - "swagger": "2.0", - "info": { - "description": "API documentation for the Fetcher Manager component", - "title": "Fetcher Manager API", - "termsOfService": "http://swagger.io/terms/", - "contact": {}, - "version": "1.0.0" - }, - "host": "localhost:4006", - "basePath": "/", - "paths": { - "/v1/fetcher": { - "post": { - "description": "Create a new data extraction job. The request will be validated, deduplicated within a 5-minute window, and all referenced connections will be tested before job creation. The metadata.source field is required for product isolation and datasource ownership validation.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Fetcher" - ], - "summary": "Create fetcher job", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "description": "Fetcher request payload. metadata.source is required.", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherRequest" - } - } - ], - "responses": { - "200": { - "description": "Duplicate request - returning existing job", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse" - } - }, - "202": { - "description": "Job created and queued for processing", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "413": { - "description": "Request Entity Too Large", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/fetcher/{id}": { - "get": { - "description": "Retrieve detailed information about a specific data extraction job.", - "produces": [ - "application/json" - ], - "tags": [ - "Fetcher" - ], - "summary": "Get fetcher job", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Job ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.JobResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections": { - "get": { - "description": "List connections with pagination and filters. When X-Product-Name is provided, returns only connections associated with that product.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "List connections", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Product name. When provided, filters connections by product.", - "name": "X-Product-Name", - "in": "header" - }, - { - "type": "integer", - "default": 1, - "description": "Page number (minimum 1)", - "name": "page", - "in": "query" - }, - { - "type": "integer", - "default": 50, - "description": "Page size (default 50, max 1000)", - "name": "limit", - "in": "query" - }, - { - "enum": [ - "asc", - "desc" - ], - "type": "string", - "default": "desc", - "description": "Sort order", - "name": "sortOrder", - "in": "query" - }, - { - "enum": [ - "ORACLE", - "SQL_SERVER", - "POSTGRESQL", - "MONGODB", - "MYSQL" - ], - "type": "string", - "description": "Filter by database type", - "name": "type", - "in": "query" - }, - { - "type": "string", - "description": "Filter by host", - "name": "host", - "in": "query" - }, - { - "type": "string", - "description": "Filter by database name", - "name": "databaseName", - "in": "query" - }, - { - "type": "string", - "description": "Filter by start date (YYYY-MM-DD)", - "name": "startDate", - "in": "query" - }, - { - "type": "string", - "description": "Filter by end date (YYYY-MM-DD)", - "name": "endDate", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "allOf": [ - { - "$ref": "#/definitions/Pagination" - }, - { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "limit": { - "type": "integer" - }, - "page": { - "type": "integer" - }, - "total": { - "type": "integer" - } - } - } - ] - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - }, - "post": { - "description": "Create a new database connection for the organization. The X-Product-Name header is required and identifies the product for this connection. ConfigName must be unique per organization (and per product when assigned); password is encrypted and a UUID is generated on creation.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Create connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Product name (required, non-empty)", - "name": "X-Product-Name", - "in": "header", - "required": true - }, - { - "description": "Connection payload", - "name": "connection", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionInput" - } - } - ], - "responses": { - "201": { - "description": "Created connection identifier", - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/unassigned": { - "get": { - "description": "List connections that have no product assigned, useful for migration purposes.", - "produces": [ - "application/json" - ], - "tags": [ - "Migration" - ], - "summary": "List unassigned connections", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "integer", - "default": 1, - "description": "Page number (minimum 1)", - "name": "page", - "in": "query" - }, - { - "type": "integer", - "default": 50, - "description": "Page size (default 50, max 1000)", - "name": "limit", - "in": "query" - }, - { - "enum": [ - "asc", - "desc" - ], - "type": "string", - "default": "desc", - "description": "Sort order", - "name": "sortOrder", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "allOf": [ - { - "$ref": "#/definitions/Pagination" - }, - { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "limit": { - "type": "integer" - }, - "page": { - "type": "integer" - }, - "total": { - "type": "integer" - } - } - } - ] - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/validate-schema": { - "post": { - "description": "Validate that tables and fields referenced in the request exist in the configured datasources. Returns 200 when validation passes, 422 when validation fails with detailed error information.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Validate schema", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "description": "Schema validation request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest" - } - } - ], - "responses": { - "200": { - "description": "Validation successful - all tables and fields exist", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse" - } - }, - "400": { - "description": "Invalid request payload or missing headers", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "422": { - "description": "Validation failed - schema errors found (missing tables, fields, or unreachable datasources)", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse" - } - }, - "500": { - "description": "Internal server error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}": { - "get": { - "description": "Get connection details by ID for the given organization.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Get connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - }, - "delete": { - "description": "Soft delete a connection when no active jobs are running for it. Returns 409 if there is any active job.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Delete connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "No Content" - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - }, - "patch": { - "description": "Apply a partial update to a connection. Only include fields you want to change. Returns 409 if there is any active job.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Update connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "Fields to update (only include fields you want to change)", - "name": "connection", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}/assign": { - "post": { - "description": "Associate an unassigned connection to a product. This is a one-time, irreversible operation for migration purposes. The product name must be provided via the X-Product-Name header.", - "produces": [ - "application/json" - ], - "tags": [ - "Migration" - ], - "summary": "Assign connection to product", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Product name to assign", - "name": "X-Product-Name", - "in": "header", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "409": { - "description": "Conflict", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}/schema": { - "get": { - "description": "Get the database schema (tables and fields) for a connection.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Get connection schema", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - }, - "/v1/management/connections/{id}/test": { - "post": { - "description": "Test the configured connection by establishing and closing a connection.", - "produces": [ - "application/json" - ], - "tags": [ - "Connections" - ], - "summary": "Test connection", - "parameters": [ - { - "type": "string", - "description": "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled.", - "name": "Authorization", - "in": "header" - }, - { - "type": "string", - "description": "Connection ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Connection test result", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "429": { - "description": "Too Many Requests", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/pkg.HTTPError" - } - } - } - } - } - }, - "definitions": { - "Pagination": { - "description": "Pagination is the struct designed to store the pagination data of an entity list.", - "type": "object", - "properties": { - "items": {}, - "limit": { - "type": "integer", - "example": 10 - }, - "page": { - "type": "integer", - "example": 1 - }, - "total": { - "type": "integer", - "example": 10 - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionInput": { - "type": "object", - "required": [ - "configName", - "databaseName", - "host", - "password", - "port", - "type", - "userName" - ], - "properties": { - "configName": { - "type": "string", - "maxLength": 100, - "minLength": 3, - "example": "production-db" - }, - "databaseName": { - "type": "string", - "example": "mydatabase" - }, - "host": { - "type": "string", - "example": "db.example.com" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "password": { - "type": "string", - "example": "secretpassword" - }, - "port": { - "type": "integer", - "maximum": 65535, - "minimum": 1, - "example": 5432 - }, - "schema": { - "type": "string", - "example": "my_schema" - }, - "ssl": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLInput" - }, - "type": { - "type": "string", - "enum": [ - "ORACLE", - "SQL_SERVER", - "POSTGRESQL", - "MONGODB", - "MYSQL" - ], - "example": "POSTGRESQL" - }, - "userName": { - "type": "string", - "example": "dbuser" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse": { - "type": "object", - "properties": { - "configName": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "host": { - "type": "string" - }, - "id": { - "type": "string" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "port": { - "type": "integer" - }, - "productName": { - "type": "string" - }, - "schema": { - "type": "string" - }, - "ssl": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLResponse" - }, - "type": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "userName": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse": { - "type": "object", - "properties": { - "configName": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "id": { - "type": "string" - }, - "tables": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.TableDetails" - } - }, - "type": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput": { - "type": "object", - "properties": { - "configName": { - "type": "string", - "maxLength": 100, - "minLength": 3, - "example": "production-db" - }, - "databaseName": { - "type": "string", - "example": "mydatabase" - }, - "host": { - "type": "string", - "example": "db.example.com" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "password": { - "type": "string", - "example": "secretpassword" - }, - "port": { - "type": "integer", - "maximum": 65535, - "minimum": 1, - "example": 5432 - }, - "schema": { - "type": "string", - "example": "my_schema" - }, - "ssl": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput" - }, - "type": { - "type": "string", - "enum": [ - "ORACLE", - "SQL_SERVER", - "POSTGRESQL", - "MONGODB", - "MYSQL" - ], - "example": "POSTGRESQL" - }, - "userName": { - "type": "string", - "example": "dbuser" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.DataRequest": { - "description": "DataRequest encapsulates field mappings and optional filters for data extraction.", - "type": "object", - "required": [ - "mappedFields" - ], - "properties": { - "filters": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.NestedFilters" - }, - "mappedFields": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.FetcherRequest": { - "description": "FetcherRequest represents the request body for creating a new data extraction job.", - "type": "object", - "required": [ - "dataRequest" - ], - "properties": { - "dataRequest": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.DataRequest" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.FetcherResponse": { - "description": "FetcherResponse represents the response after successfully creating a data extraction job.", - "type": "object", - "properties": { - "createdAt": { - "type": "string" - }, - "jobId": { - "type": "string" - }, - "message": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.JobResponse": { - "description": "JobResponse represents the complete information about a data extraction job.", - "type": "object", - "properties": { - "completedAt": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "filters": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.NestedFilters" - }, - "id": { - "type": "string" - }, - "mappedFields": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "requestHash": { - "type": "string" - }, - "resultHmac": { - "type": "string" - }, - "resultPath": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.NestedFilters": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition" - } - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SSLInput": { - "type": "object", - "properties": { - "ca": { - "type": "string", - "example": "-----BEGIN CERTIFICATE-----\n..." - }, - "cert": { - "type": "string" - }, - "key": { - "type": "string" - }, - "mode": { - "type": "string", - "example": "require" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SSLResponse": { - "type": "object", - "properties": { - "mode": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput": { - "type": "object", - "properties": { - "ca": { - "type": "string", - "example": "-----BEGIN CERTIFICATE-----\n..." - }, - "cert": { - "type": "string" - }, - "key": { - "type": "string" - }, - "mode": { - "type": "string", - "example": "require" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError": { - "type": "object", - "properties": { - "dataSourceId": { - "type": "string" - }, - "field": { - "type": "string" - }, - "table": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse": { - "type": "object", - "properties": { - "code": { - "type": "string" - }, - "errors": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError" - } - }, - "message": { - "type": "string" - }, - "title": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest": { - "description": "Request body for schema validation containing mapped fields per datasource.", - "type": "object", - "required": [ - "mappedFields" - ], - "properties": { - "mappedFields": { - "description": "MappedFields maps datasource config names to their tables and fields\nKey: configName (e.g., \"midaz_onboarding\")\nValue: map of table names to field names", - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse": { - "type": "object", - "properties": { - "errors": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError" - } - }, - "message": { - "type": "string" - }, - "status": { - "description": "\"success\" or \"failure\"", - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model.TableDetails": { - "type": "object", - "properties": { - "fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "type": "string" - } - } - }, - "github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition": { - "type": "object", - "properties": { - "between": { - "description": "Between specifies a range condition with exactly two values [min, max].\nMatches records where min \u003c= field \u003c= max\nExample: {\"between\": [100, 1000]} matches records where 100 \u003c= field \u003c= 1000", - "type": "array", - "items": {} - }, - "eq": { - "description": "Equals specifies exact value matches. Multiple values treated as OR conditions.\nExample: {\"eq\": [\"active\", \"pending\"]} matches records where field equals \"active\" OR \"pending\"", - "type": "array", - "items": {} - }, - "gt": { - "description": "GreaterThan specifies values that must be greater than the provided value.\nShould contain exactly one value for comparison.\nExample: {\"gt\": [100]} matches records where field \u003e 100", - "type": "array", - "items": {} - }, - "gte": { - "description": "GreaterOrEqual specifies values that must be greater than or equal to the provided value.\nShould contain exactly one value for comparison.\nExample: {\"gte\": [\"2025-06-01\"]} matches records where field \u003e= \"2025-06-01\"", - "type": "array", - "items": {} - }, - "in": { - "description": "In specifies a list of values where the field must match any one of them.\nMultiple values treated as OR conditions.\nExample: {\"in\": [\"active\", \"pending\", \"suspended\"]} matches any of these statuses", - "type": "array", - "items": {} - }, - "like": { - "description": "Like specifies values that must match the provided value using LIKE pattern matching.\nShould contain exactly one value for comparison.\nExample: {\"like\": [\"%active%\"]} matches any status containing \"active\"", - "type": "array", - "items": {} - }, - "lt": { - "description": "LessThan specifies values that must be less than the provided value.\nShould contain exactly one value for comparison.\nExample: {\"lt\": [1000]} matches records where field \u003c 1000", - "type": "array", - "items": {} - }, - "lte": { - "description": "LessOrEqual specifies values that must be less than or equal to the provided value.\nShould contain exactly one value for comparison.\nExample: {\"lte\": [\"2025-06-30\"]} matches records where field \u003c= \"2025-06-30\"", - "type": "array", - "items": {} - }, - "ne": { - "description": "NotEquals specifies values that must NOT match the provided value.\nShould contain exactly one value for comparison.\nExample: {\"ne\": [\"active\"]} excludes this status", - "type": "array", - "items": {} - }, - "nin": { - "description": "NotIn specifies a list of values where the field must NOT match any of them.\nMultiple values treated as AND NOT conditions.\nExample: {\"nin\": [\"deleted\", \"archived\"]} excludes these statuses", - "type": "array", - "items": {} - } - } - }, - "pkg.HTTPError": { - "type": "object", - "properties": { - "code": { - "type": "string" - }, - "entityType": { - "type": "string" - }, - "err": {}, - "message": { - "type": "string" - }, - "title": { - "type": "string" - } - } - } - } -} \ No newline at end of file diff --git a/components/manager/api/swagger.yaml b/components/manager/api/swagger.yaml deleted file mode 100644 index 0222d911..00000000 --- a/components/manager/api/swagger.yaml +++ /dev/null @@ -1,1002 +0,0 @@ -basePath: / -definitions: - Pagination: - description: Pagination is the struct designed to store the pagination data of - an entity list. - properties: - items: {} - limit: - example: 10 - type: integer - page: - example: 1 - type: integer - total: - example: 10 - type: integer - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionInput: - properties: - configName: - example: production-db - maxLength: 100 - minLength: 3 - type: string - databaseName: - example: mydatabase - type: string - host: - example: db.example.com - type: string - metadata: - additionalProperties: {} - type: object - password: - example: secretpassword - type: string - port: - example: 5432 - maximum: 65535 - minimum: 1 - type: integer - schema: - example: my_schema - type: string - ssl: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLInput' - type: - enum: - - ORACLE - - SQL_SERVER - - POSTGRESQL - - MONGODB - - MYSQL - example: POSTGRESQL - type: string - userName: - example: dbuser - type: string - required: - - configName - - databaseName - - host - - password - - port - - type - - userName - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse: - properties: - configName: - type: string - createdAt: - type: string - databaseName: - type: string - host: - type: string - id: - type: string - metadata: - additionalProperties: {} - type: object - port: - type: integer - productName: - type: string - schema: - type: string - ssl: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLResponse' - type: - type: string - updatedAt: - type: string - userName: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse: - properties: - configName: - type: string - databaseName: - type: string - id: - type: string - tables: - items: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.TableDetails' - type: array - type: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput: - properties: - configName: - example: production-db - maxLength: 100 - minLength: 3 - type: string - databaseName: - example: mydatabase - type: string - host: - example: db.example.com - type: string - metadata: - additionalProperties: {} - type: object - password: - example: secretpassword - type: string - port: - example: 5432 - maximum: 65535 - minimum: 1 - type: integer - schema: - example: my_schema - type: string - ssl: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput' - type: - enum: - - ORACLE - - SQL_SERVER - - POSTGRESQL - - MONGODB - - MYSQL - example: POSTGRESQL - type: string - userName: - example: dbuser - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.DataRequest: - description: DataRequest encapsulates field mappings and optional filters for - data extraction. - properties: - filters: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.NestedFilters' - mappedFields: - additionalProperties: - additionalProperties: - items: - type: string - type: array - type: object - type: object - required: - - mappedFields - type: object - github_com_LerianStudio_fetcher_pkg_model.FetcherRequest: - description: FetcherRequest represents the request body for creating a new data - extraction job. - properties: - dataRequest: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.DataRequest' - metadata: - additionalProperties: {} - type: object - required: - - dataRequest - type: object - github_com_LerianStudio_fetcher_pkg_model.FetcherResponse: - description: FetcherResponse represents the response after successfully creating - a data extraction job. - properties: - createdAt: - type: string - jobId: - type: string - message: - type: string - status: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.JobResponse: - description: JobResponse represents the complete information about a data extraction - job. - properties: - completedAt: - type: string - createdAt: - type: string - filters: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.NestedFilters' - id: - type: string - mappedFields: - additionalProperties: - additionalProperties: - items: - type: string - type: array - type: object - type: object - metadata: - additionalProperties: {} - type: object - requestHash: - type: string - resultHmac: - type: string - resultPath: - type: string - status: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.NestedFilters: - additionalProperties: - additionalProperties: - additionalProperties: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition' - type: object - type: object - type: object - github_com_LerianStudio_fetcher_pkg_model.SSLInput: - properties: - ca: - example: |- - -----BEGIN CERTIFICATE----- - ... - type: string - cert: - type: string - key: - type: string - mode: - example: require - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SSLResponse: - properties: - mode: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SSLUpdateInput: - properties: - ca: - example: |- - -----BEGIN CERTIFICATE----- - ... - type: string - cert: - type: string - key: - type: string - mode: - example: require - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError: - properties: - dataSourceId: - type: string - field: - type: string - table: - type: string - type: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse: - properties: - code: - type: string - errors: - items: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError' - type: array - message: - type: string - title: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest: - description: Request body for schema validation containing mapped fields per datasource. - properties: - mappedFields: - additionalProperties: - additionalProperties: - items: - type: string - type: array - type: object - description: |- - MappedFields maps datasource config names to their tables and fields - Key: configName (e.g., "midaz_onboarding") - Value: map of table names to field names - type: object - required: - - mappedFields - type: object - github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse: - properties: - errors: - items: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationError' - type: array - message: - type: string - status: - description: '"success" or "failure"' - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model.TableDetails: - properties: - fields: - items: - type: string - type: array - name: - type: string - type: object - github_com_LerianStudio_fetcher_pkg_model_job.FilterCondition: - properties: - between: - description: |- - Between specifies a range condition with exactly two values [min, max]. - Matches records where min <= field <= max - Example: {"between": [100, 1000]} matches records where 100 <= field <= 1000 - items: {} - type: array - eq: - description: |- - Equals specifies exact value matches. Multiple values treated as OR conditions. - Example: {"eq": ["active", "pending"]} matches records where field equals "active" OR "pending" - items: {} - type: array - gt: - description: |- - GreaterThan specifies values that must be greater than the provided value. - Should contain exactly one value for comparison. - Example: {"gt": [100]} matches records where field > 100 - items: {} - type: array - gte: - description: |- - GreaterOrEqual specifies values that must be greater than or equal to the provided value. - Should contain exactly one value for comparison. - Example: {"gte": ["2025-06-01"]} matches records where field >= "2025-06-01" - items: {} - type: array - in: - description: |- - In specifies a list of values where the field must match any one of them. - Multiple values treated as OR conditions. - Example: {"in": ["active", "pending", "suspended"]} matches any of these statuses - items: {} - type: array - like: - description: |- - Like specifies values that must match the provided value using LIKE pattern matching. - Should contain exactly one value for comparison. - Example: {"like": ["%active%"]} matches any status containing "active" - items: {} - type: array - lt: - description: |- - LessThan specifies values that must be less than the provided value. - Should contain exactly one value for comparison. - Example: {"lt": [1000]} matches records where field < 1000 - items: {} - type: array - lte: - description: |- - LessOrEqual specifies values that must be less than or equal to the provided value. - Should contain exactly one value for comparison. - Example: {"lte": ["2025-06-30"]} matches records where field <= "2025-06-30" - items: {} - type: array - ne: - description: |- - NotEquals specifies values that must NOT match the provided value. - Should contain exactly one value for comparison. - Example: {"ne": ["active"]} excludes this status - items: {} - type: array - nin: - description: |- - NotIn specifies a list of values where the field must NOT match any of them. - Multiple values treated as AND NOT conditions. - Example: {"nin": ["deleted", "archived"]} excludes these statuses - items: {} - type: array - type: object - pkg.HTTPError: - properties: - code: - type: string - entityType: - type: string - err: {} - message: - type: string - title: - type: string - type: object -host: localhost:4006 -info: - contact: {} - description: API documentation for the Fetcher Manager component - termsOfService: http://swagger.io/terms/ - title: Fetcher Manager API - version: 1.0.0 -paths: - /v1/fetcher: - post: - consumes: - - application/json - description: Create a new data extraction job. The request will be validated, - deduplicated within a 5-minute window, and all referenced connections will - be tested before job creation. The metadata.source field is required for product - isolation and datasource ownership validation. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Fetcher request payload. metadata.source is required. - in: body - name: request - required: true - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherRequest' - produces: - - application/json - responses: - "200": - description: Duplicate request - returning existing job - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse' - "202": - description: Job created and queued for processing - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.FetcherResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "409": - description: Conflict - schema: - $ref: '#/definitions/pkg.HTTPError' - "413": - description: Request Entity Too Large - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Create fetcher job - tags: - - Fetcher - /v1/fetcher/{id}: - get: - description: Retrieve detailed information about a specific data extraction - job. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Job ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.JobResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Get fetcher job - tags: - - Fetcher - /v1/management/connections: - get: - description: List connections with pagination and filters. When X-Product-Name - is provided, returns only connections associated with that product. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Product name. When provided, filters connections by product. - in: header - name: X-Product-Name - type: string - - default: 1 - description: Page number (minimum 1) - in: query - name: page - type: integer - - default: 50 - description: Page size (default 50, max 1000) - in: query - name: limit - type: integer - - default: desc - description: Sort order - enum: - - asc - - desc - in: query - name: sortOrder - type: string - - description: Filter by database type - enum: - - ORACLE - - SQL_SERVER - - POSTGRESQL - - MONGODB - - MYSQL - in: query - name: type - type: string - - description: Filter by host - in: query - name: host - type: string - - description: Filter by database name - in: query - name: databaseName - type: string - - description: Filter by start date (YYYY-MM-DD) - in: query - name: startDate - type: string - - description: Filter by end date (YYYY-MM-DD) - in: query - name: endDate - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - allOf: - - $ref: '#/definitions/Pagination' - - properties: - items: - items: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' - type: array - limit: - type: integer - page: - type: integer - total: - type: integer - type: object - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: List connections - tags: - - Connections - post: - consumes: - - application/json - description: Create a new database connection for the organization. The X-Product-Name - header is required and identifies the product for this connection. ConfigName - must be unique per organization (and per product when assigned); password - is encrypted and a UUID is generated on creation. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Product name (required, non-empty) - in: header - name: X-Product-Name - required: true - type: string - - description: Connection payload - in: body - name: connection - required: true - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionInput' - produces: - - application/json - responses: - "201": - description: Created connection identifier - schema: - additionalProperties: - type: string - type: object - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "409": - description: Conflict - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Create connection - tags: - - Connections - /v1/management/connections/{id}: - delete: - description: Soft delete a connection when no active jobs are running for it. - Returns 409 if there is any active job. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Connection ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "204": - description: No Content - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "409": - description: Conflict - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Delete connection - tags: - - Connections - get: - description: Get connection details by ID for the given organization. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Connection ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Get connection - tags: - - Connections - patch: - consumes: - - application/json - description: Apply a partial update to a connection. Only include fields you - want to change. Returns 409 if there is any active job. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Connection ID - in: path - name: id - required: true - type: string - - description: Fields to update (only include fields you want to change) - in: body - name: connection - required: true - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionUpdateInput' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "409": - description: Conflict - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Update connection - tags: - - Connections - /v1/management/connections/{id}/assign: - post: - description: Associate an unassigned connection to a product. This is a one-time, - irreversible operation for migration purposes. The product name must be provided - via the X-Product-Name header. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Connection ID - in: path - name: id - required: true - type: string - - description: Product name to assign - in: header - name: X-Product-Name - required: true - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "409": - description: Conflict - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Assign connection to product - tags: - - Migration - /v1/management/connections/{id}/schema: - get: - description: Get the database schema (tables and fields) for a connection. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Connection ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionSchemaResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Get connection schema - tags: - - Connections - /v1/management/connections/{id}/test: - post: - description: Test the configured connection by establishing and closing a connection. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Connection ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "200": - description: Connection test result - schema: - additionalProperties: true - type: object - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "404": - description: Not Found - schema: - $ref: '#/definitions/pkg.HTTPError' - "429": - description: Too Many Requests - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Test connection - tags: - - Connections - /v1/management/connections/unassigned: - get: - description: List connections that have no product assigned, useful for migration - purposes. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - default: 1 - description: Page number (minimum 1) - in: query - name: page - type: integer - - default: 50 - description: Page size (default 50, max 1000) - in: query - name: limit - type: integer - - default: desc - description: Sort order - enum: - - asc - - desc - in: query - name: sortOrder - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - allOf: - - $ref: '#/definitions/Pagination' - - properties: - items: - items: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.ConnectionResponse' - type: array - limit: - type: integer - page: - type: integer - total: - type: integer - type: object - "400": - description: Bad Request - schema: - $ref: '#/definitions/pkg.HTTPError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: List unassigned connections - tags: - - Migration - /v1/management/connections/validate-schema: - post: - consumes: - - application/json - description: Validate that tables and fields referenced in the request exist - in the configured datasources. Returns 200 when validation passes, 422 when - validation fails with detailed error information. - parameters: - - description: The authorization token in the 'Bearer access_token' format. - Only required when auth plugin is enabled. - in: header - name: Authorization - type: string - - description: Schema validation request - in: body - name: request - required: true - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationRequest' - produces: - - application/json - responses: - "200": - description: Validation successful - all tables and fields exist - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationResponse' - "400": - description: Invalid request payload or missing headers - schema: - $ref: '#/definitions/pkg.HTTPError' - "422": - description: Validation failed - schema errors found (missing tables, fields, - or unreachable datasources) - schema: - $ref: '#/definitions/github_com_LerianStudio_fetcher_pkg_model.SchemaValidationErrorResponse' - "500": - description: Internal server error - schema: - $ref: '#/definitions/pkg.HTTPError' - summary: Validate schema - tags: - - Connections -swagger: "2.0" diff --git a/components/manager/cmd/app/main.go b/components/manager/cmd/app/main.go index 0d28cb66..9e36eaa0 100644 --- a/components/manager/cmd/app/main.go +++ b/components/manager/cmd/app/main.go @@ -8,12 +8,6 @@ import ( "github.com/LerianStudio/fetcher/v2/pkg/startup" ) -// @title Fetcher Manager API -// @version 1.0.0 -// @description API documentation for the Fetcher Manager component -// @termsOfService http://swagger.io/terms/ -// @host localhost:4006 -// @BasePath / func main() { pkg.InitLocalEnvConfig() diff --git a/components/manager/cmd/huma-spec/main.go b/components/manager/cmd/huma-spec/main.go new file mode 100644 index 00000000..c9404a80 --- /dev/null +++ b/components/manager/cmd/huma-spec/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + + in "github.com/LerianStudio/fetcher/v2/components/manager/internal/adapters/http/in" +) + +func main() { + output := flag.String("output", "components/manager/api/openapi.yaml", "OpenAPI YAML output path, or - for stdout") + + flag.Parse() + + if *output == "-" { + spec, err := in.GenerateCanonicalSpec() + if err != nil { + log.Fatal(err) + } + + if _, err := os.Stdout.Write(spec); err != nil { + log.Fatal(err) + } + + return + } + + if err := writeSpec(*output); err != nil { + log.Fatal(err) + } +} + +func writeSpec(path string) error { + spec, err := in.GenerateCanonicalSpec() + if err != nil { + return fmt.Errorf("generate canonical OpenAPI spec: %w", err) + } + + // #nosec G306 -- api/openapi.yaml is a public, versioned artifact. + if err := os.WriteFile(path, spec, 0o644); err != nil { + return fmt.Errorf("write OpenAPI spec %s: %w", path, err) + } + + return nil +} diff --git a/components/manager/cmd/huma-spec/main_test.go b/components/manager/cmd/huma-spec/main_test.go new file mode 100644 index 00000000..33df1839 --- /dev/null +++ b/components/manager/cmd/huma-spec/main_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + in "github.com/LerianStudio/fetcher/v2/components/manager/internal/adapters/http/in" + + "github.com/stretchr/testify/require" +) + +func TestWriteSpec_WritesCanonicalBytes(t *testing.T) { + path := filepath.Join(t.TempDir(), "openapi.yaml") + require.NoError(t, writeSpec(path)) + + got, err := os.ReadFile(path) + require.NoError(t, err) + want, err := in.GenerateCanonicalSpec() + require.NoError(t, err) + require.Equal(t, want, got) +} + +func TestCommittedSpec_IsCurrent(t *testing.T) { + committed, err := os.ReadFile(filepath.Join("..", "..", "api", "openapi.yaml")) + require.NoError(t, err) + want, err := in.GenerateCanonicalSpec() + require.NoError(t, err) + + require.Equal(t, want, committed, "run `make generate-docs`") +} diff --git a/components/manager/internal/adapters/http/in/connection.go b/components/manager/internal/adapters/http/in/connection.go index cdb59ac2..c1594155 100644 --- a/components/manager/internal/adapters/http/in/connection.go +++ b/components/manager/internal/adapters/http/in/connection.go @@ -1,24 +1,8 @@ package in import ( - "fmt" - - "github.com/LerianStudio/lib-observability" - "github.com/LerianStudio/fetcher/v2/components/manager/internal/services/command" "github.com/LerianStudio/fetcher/v2/components/manager/internal/services/query" - - "github.com/LerianStudio/fetcher/v2/pkg" - "github.com/LerianStudio/fetcher/v2/pkg/constant" - "github.com/LerianStudio/fetcher/v2/pkg/model" - httpUtils "github.com/LerianStudio/fetcher/v2/pkg/net/http" - - libLog "github.com/LerianStudio/lib-observability/log" - libOpentelemetry "github.com/LerianStudio/lib-observability/tracing" - - "github.com/gofiber/fiber/v2" - "github.com/google/uuid" - "go.opentelemetry.io/otel/attribute" ) type ConnectionHandler struct { @@ -53,530 +37,3 @@ func NewConnectionHandler( GetSchemaQuery: getSchemaQuery, } } - -// CreateConnection is a method that creates a database connection. -// -// @Summary Create connection -// @Description Create a new database connection for the organization. The X-Product-Name header is required and identifies the product for this connection. ConfigName must be unique per organization (and per product when assigned); password is encrypted and a UUID is generated on creation. -// @Tags Connections -// @Accept json -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param X-Product-Name header string true "Product name (required, non-empty)" -// @Param connection body model.ConnectionInput true "Connection payload" -// @Success 201 {object} map[string]string "Created connection identifier" -// @Failure 400 {object} pkg.HTTPError -// @Failure 409 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections [post] -func (h *ConnectionHandler) CreateConnection(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.create_connection") - defer span.End() - - c.SetUserContext(ctx) - - productName, err := httpUtils.GetRequiredProductName(c) - if err != nil { - libOpentelemetry.HandleSpanError(span, "missing or invalid product name", err) - return httpUtils.WithError(c, err) - } - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - attribute.String("app.request.product_name", productName), - ) - - var request model.ConnectionInput - if errParser := c.BodyParser(&request); errParser != nil { - libOpentelemetry.HandleSpanError(span, "failed to parse payload", errParser) - - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrBadRequest.Error(), - Title: "Invalid payload", - Message: "unable to parse request body", - Err: errParser, - }) - } - - if request.IsEmpty() { - err := pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrBadRequest.Error(), - Title: "Invalid payload", - Message: "empty request body", - } - - libOpentelemetry.HandleSpanError(span, "empty request body", err) - - return httpUtils.WithError(c, err) - } - - // Struct-tag validation (required, hostname|ip, safe_host, ...). Without - // this call the `safe_host` tag is dead code — BodyParser does not invoke - // the validator. See docs/PROJECT_RULES.md § "Defense-in-Depth: Two Layers". - if err := httpUtils.ValidateStruct(&request); err != nil { - libOpentelemetry.HandleSpanError(span, "request validation failed", err) - - return httpUtils.WithError(c, err) - } - - conn, err := h.CreateCmd.Execute(ctx, request, productName) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute create connection command, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to create connection", err) - - return httpUtils.WithError(c, err) - } - - resp := model.NewConnectionResponseFrom(conn) - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connection created id=%s", resp.ID)) - - return httpUtils.Created(c, resp) -} - -// ListConnections is a method that retrieves connections with optional pagination and filters. -// -// @Summary List connections -// @Description List connections with pagination and filters. When X-Product-Name is provided, returns only connections associated with that product. -// @Tags Connections -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param X-Product-Name header string false "Product name. When provided, filters connections by product." -// @Param page query int false "Page number (minimum 1)" default(1) -// @Param limit query int false "Page size (default 50, max 1000)" default(50) -// @Param sortOrder query string false "Sort order" Enums(asc, desc) default(desc) -// @Param type query string false "Filter by database type" Enums(ORACLE, SQL_SERVER, POSTGRESQL, MONGODB, MYSQL) -// @Param host query string false "Filter by host" -// @Param databaseName query string false "Filter by database name" -// @Param startDate query string false "Filter by start date (YYYY-MM-DD)" -// @Param endDate query string false "Filter by end date (YYYY-MM-DD)" -// @Success 200 {object} model.Pagination{items=[]model.ConnectionResponse,page=int,limit=int,total=int} -// @Failure 400 {object} pkg.HTTPError -// @Failure 404 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections [get] -func (h *ConnectionHandler) ListConnections(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.list_connection") - defer span.End() - - c.SetUserContext(ctx) - - productName, err := httpUtils.GetProductName(c) - if err != nil { - libOpentelemetry.HandleSpanError(span, "invalid product name", err) - return httpUtils.WithError(c, err) - } - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - if productName != "" { - span.SetAttributes(attribute.String("app.request.product_name", productName)) - } - - headerParams, err := httpUtils.ValidateParameters(c.Queries()) - if err != nil { - libOpentelemetry.HandleSpanError(span, "Failed to validate query parameters", err) - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to validate query parameters, Error: %s", err.Error())) - - return httpUtils.WithError(c, err) - } - - pagination, err := h.ListQuery.Execute(ctx, productName, *headerParams) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute list connections query, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to list connections", err) - - return httpUtils.WithError(c, err) - } - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connections listed count=%d", pagination.Total)) - - return httpUtils.OK(c, pagination) -} - -// GetConnection is a method that retrieves a connection by ID. -// -// @Summary Get connection -// @Description Get connection details by ID for the given organization. -// @Tags Connections -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param id path string true "Connection ID" -// @Success 200 {object} model.ConnectionResponse -// @Failure 400 {object} pkg.HTTPError -// @Failure 404 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections/{id} [get] -func (h *ConnectionHandler) GetConnection(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.get_connection") - defer span.End() - - c.SetUserContext(ctx) - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - id, err := uuid.Parse(c.Params("id")) - if err != nil { - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrInvalidPathParameter.Error(), - Title: "Invalid Path Parameter", - Message: "invalid connection id", - Err: err, - }) - } - - span.SetAttributes(attribute.String("app.request.connection_id", id.String())) - - conn, err := h.GetQuery.Execute(ctx, id) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute get connection query, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to get connection", err) - - return httpUtils.WithError(c, err) - } - - resp := model.NewConnectionResponseFrom(conn) - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connection retrieved id=%s", id)) - - return httpUtils.OK(c, resp) -} - -// TestConnection tests a database connection by attempting to connect and disconnect. -// -// @Summary Test connection -// @Description Test the configured connection by establishing and closing a connection. -// @Tags Connections -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param id path string true "Connection ID" -// @Success 200 {object} map[string]any "Connection test result" -// @Failure 400 {object} pkg.HTTPError -// @Failure 404 {object} pkg.HTTPError -// @Failure 429 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections/{id}/test [post] -func (h *ConnectionHandler) TestConnection(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.test_connection") - defer span.End() - - c.SetUserContext(ctx) - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - id, err := uuid.Parse(c.Params("id")) - if err != nil { - libOpentelemetry.HandleSpanError(span, "invalid connection id parameter", err) - - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrInvalidPathParameter.Error(), - Title: "Invalid Path Parameter", - Message: "invalid connection id", - Err: err, - }) - } - - span.SetAttributes(attribute.String("app.request.connection_id", id.String())) - - resp, err := h.TestQuery.Execute(ctx, id) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute test connection query, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to test connection", err) - - return httpUtils.WithError(c, err) - } - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connection test successful id=%s latency_ms=%d", id, resp.LatencyMs)) - - return httpUtils.OK(c, resp) -} - -// UpdateConnection is a method that partially updates a connection. -// -// @Summary Update connection -// @Description Apply a partial update to a connection. Only include fields you want to change. Returns 409 if there is any active job. -// @Tags Connections -// @Accept json -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param id path string true "Connection ID" -// @Param connection body model.ConnectionUpdateInput true "Fields to update (only include fields you want to change)" -// @Success 200 {object} model.ConnectionResponse -// @Failure 400 {object} pkg.HTTPError -// @Failure 404 {object} pkg.HTTPError -// @Failure 409 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections/{id} [patch] -func (h *ConnectionHandler) UpdateConnection(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.update_connection") - defer span.End() - - c.SetUserContext(ctx) - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - id, err := uuid.Parse(c.Params("id")) - if err != nil { - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrInvalidPathParameter.Error(), - Title: "Invalid Path Parameter", - Message: "invalid connection id", - Err: err, - }) - } - - span.SetAttributes(attribute.String("app.request.connection_id", id.String())) - - var request model.ConnectionUpdateInput - if errParser := c.BodyParser(&request); errParser != nil { - libOpentelemetry.HandleSpanError(span, "failed to parse payload", errParser) - - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrBadRequest.Error(), - Title: "Invalid payload", - Message: "unable to parse request body", - Err: errParser, - }) - } - - if request.IsEmpty() { - err := pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrBadRequest.Error(), - Title: "Invalid payload", - Message: "empty request body", - } - - libOpentelemetry.HandleSpanError(span, "empty request body", err) - - return httpUtils.WithError(c, err) - } - - // Struct-tag validation (omitempty, hostname|ip, safe_host, ...). Required - // so the `safe_host` tag in ConnectionUpdateInput is actually enforced. - if err := httpUtils.ValidateStruct(&request); err != nil { - libOpentelemetry.HandleSpanError(span, "request validation failed", err) - - return httpUtils.WithError(c, err) - } - - conn, err := h.UpdateCmd.Execute(ctx, id, request) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute update connection command, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to update connection", err) - - return httpUtils.WithError(c, err) - } - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connection updated id=%s", id)) - - return httpUtils.OK(c, model.NewConnectionResponseFrom(conn)) -} - -// DeleteConnection is a method that performs a soft delete of a connection. -// -// @Summary Delete connection -// @Description Soft delete a connection when no active jobs are running for it. Returns 409 if there is any active job. -// @Tags Connections -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param id path string true "Connection ID" -// @Success 204 "No Content" -// @Failure 400 {object} pkg.HTTPError -// @Failure 404 {object} pkg.HTTPError -// @Failure 409 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections/{id} [delete] -func (h *ConnectionHandler) DeleteConnection(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.delete_connection") - defer span.End() - - c.SetUserContext(ctx) - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - id, err := uuid.Parse(c.Params("id")) - if err != nil { - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrInvalidPathParameter.Error(), - Title: "Invalid Path Parameter", - Message: "invalid connection id", - Err: err, - }) - } - - span.SetAttributes(attribute.String("app.request.connection_id", id.String())) - - if err := h.DeleteCmd.Execute(ctx, id); err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute delete connection command, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to delete connection", err) - - return httpUtils.WithError(c, err) - } - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connection deleted id=%s", id)) - - return c.Status(fiber.StatusNoContent).Send(nil) -} - -// ValidateSchema validates schema references against configured datasources. -// -// Returns 200 OK with SchemaValidationResponse when all tables and fields exist. -// Returns 422 Unprocessable Entity with SchemaValidationErrorResponse when any -// datasource is not found, table doesn't exist, field is missing, or datasource is unreachable. -// -// @Summary Validate schema -// @Description Validate that tables and fields referenced in the request exist in the configured datasources. Returns 200 when validation passes, 422 when validation fails with detailed error information. -// @Tags Connections -// @Accept json -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param request body model.SchemaValidationRequest true "Schema validation request" -// @Success 200 {object} model.SchemaValidationResponse "Validation successful - all tables and fields exist" -// @Failure 400 {object} pkg.HTTPError "Invalid request payload or missing headers" -// @Failure 422 {object} model.SchemaValidationErrorResponse "Validation failed - schema errors found (missing tables, fields, or unreachable datasources)" -// @Failure 500 {object} pkg.HTTPError "Internal server error" -// @Router /v1/management/connections/validate-schema [post] -func (h *ConnectionHandler) ValidateSchema(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.validate_schema") - defer span.End() - - c.SetUserContext(ctx) - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - var request model.SchemaValidationRequest - if errParser := c.BodyParser(&request); errParser != nil { - libOpentelemetry.HandleSpanError(span, "failed to parse payload", errParser) - - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "schema", - Code: constant.ErrBadRequest.Error(), - Title: "Invalid payload", - Message: "unable to parse request body", - Err: errParser, - }) - } - - // Struct-tag validation — enforces `required` on MappedFields consistently - // with the sibling connection handlers. - if err := httpUtils.ValidateStruct(&request); err != nil { - libOpentelemetry.HandleSpanError(span, "request validation failed", err) - - return httpUtils.WithError(c, err) - } - - resp, err := h.ValidateSchemaQuery.Execute(ctx, request) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute validate schema query, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to validate schema", err) - - return httpUtils.WithError(c, err) - } - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("schema validation completed status=%s", resp.Status)) - - if resp.Status == model.StatusFailure { - return httpUtils.JSONResponse(c, fiber.StatusUnprocessableEntity, model.SchemaValidationErrorResponse{ - Title: "Schema validation failed", - Code: constant.ErrSchemaValidationFailed.Error(), - Message: resp.Message, - Errors: resp.Errors, - }) - } - - return httpUtils.OK(c, resp) -} - -// GetConnectionSchema retrieves the database schema for a connection. -// -// @Summary Get connection schema -// @Description Get the database schema (tables and fields) for a connection. -// @Tags Connections -// @Produce json -// @Param Authorization header string false "The authorization token in the 'Bearer access_token' format. Only required when auth plugin is enabled." -// @Param id path string true "Connection ID" -// @Success 200 {object} model.ConnectionSchemaResponse -// @Failure 400 {object} pkg.HTTPError -// @Failure 404 {object} pkg.HTTPError -// @Failure 500 {object} pkg.HTTPError -// @Router /v1/management/connections/{id}/schema [get] -func (h *ConnectionHandler) GetConnectionSchema(c *fiber.Ctx) error { - ctx := c.UserContext() - logger, tracer, reqID, _ := observability.NewTrackingFromContext(ctx) - - ctx, span := tracer.Start(ctx, "handler.get_connection_schema") - defer span.End() - - c.SetUserContext(ctx) - - span.SetAttributes( - attribute.String("app.request.request_id", reqID), - ) - - id, err := uuid.Parse(c.Params("id")) - if err != nil { - libOpentelemetry.HandleSpanError(span, "invalid connection id parameter", err) - - return httpUtils.WithError(c, pkg.ValidationError{ - EntityType: "connection", - Code: constant.ErrInvalidPathParameter.Error(), - Title: "Invalid Path Parameter", - Message: "invalid connection id", - Err: err, - }) - } - - span.SetAttributes(attribute.String("app.request.connection_id", id.String())) - - resp, err := h.GetSchemaQuery.Execute(ctx, id) - if err != nil { - logger.Log(ctx, libLog.LevelError, fmt.Sprintf("Failed to execute get connection schema query, Error: %s", err.Error())) - libOpentelemetry.HandleSpanError(span, "failed to get connection schema", err) - - return httpUtils.WithError(c, err) - } - - logger.Log(ctx, libLog.LevelInfo, fmt.Sprintf("connection schema retrieved id=%s tables=%d", id, len(resp.Tables))) - - return httpUtils.OK(c, resp) -} diff --git a/components/manager/internal/adapters/http/in/connection_huma_test_helpers_test.go b/components/manager/internal/adapters/http/in/connection_huma_test_helpers_test.go new file mode 100644 index 00000000..0edac46a --- /dev/null +++ b/components/manager/internal/adapters/http/in/connection_huma_test_helpers_test.go @@ -0,0 +1,168 @@ +package in + +import ( + "context" + "testing" + "time" + + "github.com/LerianStudio/fetcher/pkg/engine" + "github.com/LerianStudio/fetcher/v2/components/manager/internal/services/command" + "github.com/LerianStudio/fetcher/v2/components/manager/internal/services/query" + pkgdatasource "github.com/LerianStudio/fetcher/v2/pkg/datasource" + "github.com/LerianStudio/fetcher/v2/pkg/enginecompat/schemacompat" + "github.com/LerianStudio/fetcher/v2/pkg/model" + cacheRepo "github.com/LerianStudio/fetcher/v2/pkg/ports/cache" + "github.com/LerianStudio/fetcher/v2/pkg/testutil" + observability "github.com/LerianStudio/lib-observability" + libLog "github.com/LerianStudio/lib-observability/log" + + "github.com/gofiber/fiber/v2" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" +) + +func setupConnectionTestApp() *fiber.App { + app := fiber.New(fiber.Config{BodyLimit: 10 * 1024}) + app.Use(func(c *fiber.Ctx) error { + logger := &libLog.GoLogger{Level: libLog.LevelDebug} + ctx := observability.ContextWithHeaderID(testutil.TestContext(), "test-request-id") + ctx = observability.ContextWithLogger(ctx, logger) + ctx = observability.ContextWithTracer(ctx, otel.Tracer("test")) + c.SetUserContext(ctx) + + return c.Next() + }) + + return app +} + +func createTestConnection(id uuid.UUID) *model.Connection { + now := time.Now().UTC() + + return &model.Connection{ + ID: id, + ProductName: "test-product", + ConfigName: "test-connection", + Type: model.TypePostgreSQL, + Host: "localhost", + Port: 5432, + DatabaseName: "testdb", + Username: "testuser", + PasswordEncrypted: "encrypted-password", + EncryptionKeyVersion: "v1", + CreatedAt: now, + UpdatedAt: now, + } +} + +func validConnectionInput() string { + return `{ + "configName": "test-connection", + "type": "POSTGRESQL", + "host": "localhost", + "port": 5432, + "databaseName": "testdb", + "userName": "testuser", + "password": "secretpassword" + }` +} + +func validSchemaValidationRequest() string { + return `{ + "mappedFields": { + "ds1": { + "table1": ["field1", "field2"], + "table2": ["field3"] + } + } + }` +} + +func newSchemaEngineForTest( + t *testing.T, + factory pkgdatasource.DataSourceFactory, + cache cacheRepo.SchemaCacheRepository, +) *engine.Engine { + t.Helper() + + eng, err := engine.New( + engine.WithConnectorRegistry(schemaConnectorRegistryForTest{factory: schemacompat.NewConnectorFactory(factory, nil)}), + engine.WithConnectionStore(schemacompat.NewConnectionStore()), + engine.WithSchemaCache(schemacompat.NewSchemaCache(cache, 0)), + ) + require.NoError(t, err) + + return eng +} + +type schemaConnectorRegistryForTest struct { + factory engine.ConnectorFactory +} + +func (r schemaConnectorRegistryForTest) Connector(string) (engine.ConnectorFactory, bool) { + return r.factory, true +} + +type noopSchemaCache struct{} + +func newNoopSchemaCache() *noopSchemaCache { + return &noopSchemaCache{} +} + +func (n *noopSchemaCache) Get(_ context.Context, _ string) (*model.DataSourceSchema, error) { + return nil, nil +} + +func (n *noopSchemaCache) Set(_ context.Context, _ string, _ *model.DataSourceSchema, _ time.Duration) error { + return nil +} + +func (n *noopSchemaCache) Delete(_ context.Context, _ string) error { + return nil +} + +func (n *noopSchemaCache) Clear(_ context.Context) error { + return nil +} + +func (n *noopSchemaCache) IsHealthy(_ context.Context) bool { + return true +} + +func (n *noopSchemaCache) Close() error { + return nil +} + +func TestNewConnectionHandlerPreservesDependencies(t *testing.T) { + createCmd := command.NewCreateConnection(nil, nil) + updateCmd := command.NewUpdateConnection(nil, nil) + deleteCmd := command.NewDeleteConnection(nil) + getQuery := query.NewGetConnection(nil, nil, nil) + listQuery := query.NewListConnections(nil, nil) + testQuery := query.NewTestConnection(nil, nil, nil, nil, nil, nil) + validateSchemaQuery := query.NewValidateSchema(nil, nil, nil) + getSchemaQuery := query.NewGetConnectionSchema(nil, nil, nil, nil, false) + + handler := NewConnectionHandler( + createCmd, + updateCmd, + deleteCmd, + getQuery, + listQuery, + testQuery, + validateSchemaQuery, + getSchemaQuery, + ) + + assert.NotNil(t, handler) + assert.Same(t, createCmd, handler.CreateCmd) + assert.Same(t, updateCmd, handler.UpdateCmd) + assert.Same(t, deleteCmd, handler.DeleteCmd) + assert.Same(t, getQuery, handler.GetQuery) + assert.Same(t, listQuery, handler.ListQuery) + assert.Same(t, testQuery, handler.TestQuery) + assert.Same(t, validateSchemaQuery, handler.ValidateSchemaQuery) + assert.Same(t, getSchemaQuery, handler.GetSchemaQuery) +} diff --git a/components/manager/internal/adapters/http/in/connection_test.go b/components/manager/internal/adapters/http/in/connection_test.go deleted file mode 100644 index 7f8eb44b..00000000 --- a/components/manager/internal/adapters/http/in/connection_test.go +++ /dev/null @@ -1,1394 +0,0 @@ -package in - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/LerianStudio/fetcher/pkg/engine" - "github.com/LerianStudio/fetcher/v2/components/manager/internal/services/command" - "github.com/LerianStudio/fetcher/v2/components/manager/internal/services/query" - pkgdatasource "github.com/LerianStudio/fetcher/v2/pkg/datasource" - "github.com/LerianStudio/fetcher/v2/pkg/enginecompat/schemacompat" - "github.com/LerianStudio/fetcher/v2/pkg/model" - "github.com/LerianStudio/fetcher/v2/pkg/model/datasource" - jobRepo "github.com/LerianStudio/fetcher/v2/pkg/mongodb/job" - cacheRepo "github.com/LerianStudio/fetcher/v2/pkg/ports/cache" - connRepo "github.com/LerianStudio/fetcher/v2/pkg/ports/connection" - "github.com/LerianStudio/fetcher/v2/pkg/testutil" - observability "github.com/LerianStudio/lib-observability" - - "github.com/LerianStudio/fetcher/v2/pkg/crypto" - - libLog "github.com/LerianStudio/lib-observability/log" - "github.com/gofiber/fiber/v2" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel" - "go.uber.org/mock/gomock" -) - -// setupConnectionTestApp creates a Fiber app with test context middleware for connection tests. -func setupConnectionTestApp() *fiber.App { - app := fiber.New(fiber.Config{ - BodyLimit: 10 * 1024, // 10KB for test flexibility - }) - - // Middleware to inject test context with logger and tracer - app.Use(func(c *fiber.Ctx) error { - logger := &libLog.GoLogger{Level: libLog.LevelDebug} - ctx := observability.ContextWithHeaderID(testutil.TestContext(), "test-request-id") - ctx = observability.ContextWithLogger(ctx, logger) - ctx = observability.ContextWithTracer(ctx, otel.Tracer("test")) - c.SetUserContext(ctx) - - return c.Next() - }) - - return app -} - -// createTestConnection creates a test connection with default values. -func createTestConnection(id uuid.UUID) *model.Connection { - now := time.Now().UTC() - return &model.Connection{ - ID: id, - ProductName: "test-product", - ConfigName: "test-connection", - Type: model.TypePostgreSQL, - Host: "localhost", - Port: 5432, - DatabaseName: "testdb", - Username: "testuser", - PasswordEncrypted: "encrypted-password", - EncryptionKeyVersion: "v1", - CreatedAt: now, - UpdatedAt: now, - } -} - -// validConnectionInput returns a valid ConnectionInput for tests. -func validConnectionInput() string { - return `{ - "configName": "test-connection", - "type": "POSTGRESQL", - "host": "localhost", - "port": 5432, - "databaseName": "testdb", - "username": "testuser", - "password": "secretpassword" - }` -} - -// ============================================================================ -// CreateConnection Handler Tests -// ============================================================================ - -func TestConnectionHandler_CreateConnection_Success(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockConnRepo := connRepo.NewMockRepository(ctrl) - mockCryptor := crypto.NewMockCryptor(ctrl) - - connID := uuid.New() - - testConn := createTestConnection(connID) - - // Mock expectations for CreateConnection service: - // 1. Encrypt password (called by model.NewConnection) - mockCryptor.EXPECT().Encrypt(gomock.Any(), "secretpassword").Return("encrypted-password", "v1", nil) - // 2. Check for duplicate config name - mockConnRepo.EXPECT().FindByName(gomock.Any(), "test-connection").Return(nil, nil) - // 3. Create connection - mockConnRepo.EXPECT().Create(gomock.Any(), gomock.Any()).Return(testConn, nil) - - createCmd := command.NewCreateConnection(mockCryptor, connectionEngineForConnRepo(t, mockConnRepo, nil)) - handler := &ConnectionHandler{CreateCmd: createCmd} - - app := setupConnectionTestApp() - app.Post("/v1/management/connections", handler.CreateConnection) - - req := httptest.NewRequest("POST", "/v1/management/connections", strings.NewReader(validConnectionInput())) - req.Header.Set("Content-Type", "application/json") - - req.Header.Set("X-Product-Name", "test-product") - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, fiber.StatusCreated, resp.StatusCode) - - var body map[string]interface{} - err = json.NewDecoder(resp.Body).Decode(&body) - require.NoError(t, err) - assert.NotEmpty(t, body["id"]) -} - -func TestConnectionHandler_CreateConnection_InvalidJSON(t *testing.T) { - app := setupConnectionTestApp() - - handler := &ConnectionHandler{CreateCmd: nil} - app.Post("/v1/management/connections", handler.CreateConnection) - - tests := []struct { - name string - body string - wantCode int - }{ - { - name: "invalid JSON - missing closing brace", - body: `{"configName": "test"`, - wantCode: fiber.StatusBadRequest, - }, - { - name: "invalid JSON - syntax error", - body: `{invalid}`, - wantCode: fiber.StatusBadRequest, - }, - { - name: "invalid JSON - empty string", - body: ``, - wantCode: fiber.StatusBadRequest, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest("POST", "/v1/management/connections", strings.NewReader(tt.body)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("X-Organization-Id", uuid.New().String()) - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, tt.wantCode, resp.StatusCode) - }) - } -} - -func TestConnectionHandler_CreateConnection_MissingProductNameHeader(t *testing.T) { - app := setupConnectionTestApp() - - handler := &ConnectionHandler{CreateCmd: nil} - app.Post("/v1/management/connections", handler.CreateConnection) - - tests := []struct { - name string - productHeader string - setHeader bool - wantCode int - }{ - { - name: "missing X-Product-Name header", - productHeader: "", - setHeader: false, - wantCode: fiber.StatusBadRequest, - }, - { - name: "empty X-Product-Name header", - productHeader: "", - setHeader: true, - wantCode: fiber.StatusBadRequest, - }, - { - name: "whitespace only X-Product-Name header", - productHeader: " ", - setHeader: true, - wantCode: fiber.StatusBadRequest, - }, - { - name: "invalid characters in X-Product-Name header", - productHeader: "my product!", - setHeader: true, - wantCode: fiber.StatusBadRequest, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest("POST", "/v1/management/connections", strings.NewReader(validConnectionInput())) - req.Header.Set("Content-Type", "application/json") - - if tt.setHeader { - req.Header.Set("X-Product-Name", tt.productHeader) - } - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, tt.wantCode, resp.StatusCode) - }) - } -} - -func TestConnectionHandler_CreateConnection_Conflict(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockConnRepo := connRepo.NewMockRepository(ctrl) - mockCryptor := crypto.NewMockCryptor(ctrl) - - existingConn := createTestConnection(uuid.New()) - - // Service will encrypt, then find existing connection -> conflict - mockCryptor.EXPECT().Encrypt(gomock.Any(), "secretpassword").Return("encrypted-password", "v1", nil) - mockConnRepo.EXPECT().FindByName(gomock.Any(), "test-connection").Return(existingConn, nil) - - createCmd := command.NewCreateConnection(mockCryptor, connectionEngineForConnRepo(t, mockConnRepo, nil)) - handler := &ConnectionHandler{CreateCmd: createCmd} - - app := setupConnectionTestApp() - app.Post("/v1/management/connections", handler.CreateConnection) - - req := httptest.NewRequest("POST", "/v1/management/connections", strings.NewReader(validConnectionInput())) - req.Header.Set("Content-Type", "application/json") - - req.Header.Set("X-Product-Name", "test-product") - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, fiber.StatusConflict, resp.StatusCode) -} - -func TestConnectionHandler_CreateConnection_InternalError(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockConnRepo := connRepo.NewMockRepository(ctrl) - mockCryptor := crypto.NewMockCryptor(ctrl) - - // Service will encrypt, check dupe, then fail on create - mockCryptor.EXPECT().Encrypt(gomock.Any(), "secretpassword").Return("encrypted-password", "v1", nil) - mockConnRepo.EXPECT().FindByName(gomock.Any(), "test-connection").Return(nil, nil) - mockConnRepo.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, assert.AnError) - - createCmd := command.NewCreateConnection(mockCryptor, connectionEngineForConnRepo(t, mockConnRepo, nil)) - handler := &ConnectionHandler{CreateCmd: createCmd} - - app := setupConnectionTestApp() - app.Post("/v1/management/connections", handler.CreateConnection) - - req := httptest.NewRequest("POST", "/v1/management/connections", strings.NewReader(validConnectionInput())) - req.Header.Set("Content-Type", "application/json") - - req.Header.Set("X-Product-Name", "test-product") - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, fiber.StatusInternalServerError, resp.StatusCode) -} - -// ============================================================================ -// GetConnection Handler Tests -// ============================================================================ - -func TestConnectionHandler_GetConnection_Success(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockConnRepo := connRepo.NewMockRepository(ctrl) - - connID := uuid.New() - - testConn := createTestConnection(connID) - - mockConnRepo.EXPECT().FindByID(gomock.Any(), connID).Return(testConn, nil) - - getQuery := query.NewGetConnection(nil, nil, scopeAuthorityEngine(t, mockConnRepo)) - handler := &ConnectionHandler{GetQuery: getQuery} - - app := setupConnectionTestApp() - app.Get("/v1/management/connections/:id", handler.GetConnection) - - req := httptest.NewRequest("GET", "/v1/management/connections/"+connID.String(), nil) - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, fiber.StatusOK, resp.StatusCode) - - var body model.ConnectionResponse - err = json.NewDecoder(resp.Body).Decode(&body) - require.NoError(t, err) - assert.Equal(t, connID, body.ID) - assert.Equal(t, "test-connection", body.ConfigName) -} - -func TestConnectionHandler_GetConnection_NotFound(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockConnRepo := connRepo.NewMockRepository(ctrl) - - connID := uuid.New() - - // Service returns nil for not found - mockConnRepo.EXPECT().FindByID(gomock.Any(), connID).Return(nil, nil) - - getQuery := query.NewGetConnection(nil, nil, scopeAuthorityEngine(t, mockConnRepo)) - handler := &ConnectionHandler{GetQuery: getQuery} - - app := setupConnectionTestApp() - app.Get("/v1/management/connections/:id", handler.GetConnection) - - req := httptest.NewRequest("GET", "/v1/management/connections/"+connID.String(), nil) - - resp, err := app.Test(req) - require.NoError(t, err) - defer resp.Body.Close() - - assert.Equal(t, fiber.StatusNotFound, resp.StatusCode) -} - -func TestConnectionHandler_GetConnection_InvalidID(t *testing.T) { - app := setupConnectionTestApp() - - handler := &ConnectionHandler{GetQuery: nil} - app.Get("/v1/management/connections/:id", handler.GetConnection) - - tests := []struct { - name string - connID string - wantCode int - }{ - { - name: "invalid UUID format", - connID: "not-a-uuid", - wantCode: fiber.StatusBadRequest, - }, - { - name: "partial UUID", - connID: "550e8400-e29b", - wantCode: fiber.StatusBadRequest, - }, - { - name: "UUID with special characters", - connID: "550e8400-e29b-