diff --git a/src/auth/commands/setup.ts b/src/auth/commands/setup.ts index 0aeb917..4831305 100644 --- a/src/auth/commands/setup.ts +++ b/src/auth/commands/setup.ts @@ -1,6 +1,7 @@ import { stdin as input, stdout as output } from "node:process"; import { createInterface } from "node:readline/promises"; import { Command } from "commander"; +import { getRootOpts } from "../../cli/services/global-opts.js"; import { CliUsageError, handleCliError, @@ -8,7 +9,6 @@ import { import { validateAuthToken } from "../../scrape/services/auth-validation.js"; import { PLAYGROUND_URL } from "../constants.js"; import { getConfigPath, writeConfig } from "../services/config.js"; -import { getRootOpts } from "../services/global-opts.js"; const TOKEN_PROMPT = `Paste your Web Scraping API basic auth token (${PLAYGROUND_URL}): `; diff --git a/src/auth/commands/whoami.ts b/src/auth/commands/whoami.ts index d55405c..c63d9ec 100644 --- a/src/auth/commands/whoami.ts +++ b/src/auth/commands/whoami.ts @@ -1,7 +1,7 @@ import { Command } from "commander"; +import { getRootOpts } from "../../cli/services/global-opts.js"; import { handleCliError } from "../../platform/services/handle-cli-error.js"; import { AuthRequiredError } from "../errors/auth-required-error.js"; -import { getRootOpts } from "../services/global-opts.js"; import { mask } from "../services/mask.js"; import { resolveAuthToken } from "../services/resolve-token.js"; diff --git a/src/auth/services/global-opts.ts b/src/auth/services/global-opts.ts deleted file mode 100644 index eaee402..0000000 --- a/src/auth/services/global-opts.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Command } from "commander"; - -export function getRootOpts(command: Command): { token?: string } { - let current: Command = command; - - while (current.parent) { - current = current.parent; - } - - return current.opts() as { token?: string }; -} diff --git a/src/auth/services/resolve-token.ts b/src/auth/services/resolve-token.ts index fd6285a..c9a59d8 100644 --- a/src/auth/services/resolve-token.ts +++ b/src/auth/services/resolve-token.ts @@ -1,4 +1,3 @@ -import { AuthRequiredError } from "../errors/auth-required-error.js"; import { readConfig } from "./config.js"; export type AuthSource = "flag" | "env" | "config" | "none"; @@ -33,15 +32,3 @@ export async function resolveAuthToken( return { token: undefined, source: "none" }; } - -export async function requireAuthToken( - options: ResolveAuthOptions = {} -): Promise { - const { token } = await resolveAuthToken(options); - - if (!token) { - throw new AuthRequiredError(); - } - - return token; -} diff --git a/src/cli/services/global-opts.ts b/src/cli/services/global-opts.ts new file mode 100644 index 0000000..3867460 --- /dev/null +++ b/src/cli/services/global-opts.ts @@ -0,0 +1,16 @@ +import type { Command } from "commander"; + +export interface RootOptions { + token?: string; + verbose?: boolean; +} + +export function getRootOpts(command: Command): RootOptions { + let current: Command = command; + + while (current.parent) { + current = current.parent; + } + + return current.opts() as RootOptions; +} diff --git a/src/cli/services/verbose-log.ts b/src/cli/services/verbose-log.ts new file mode 100644 index 0000000..2d7b5fc --- /dev/null +++ b/src/cli/services/verbose-log.ts @@ -0,0 +1,7 @@ +export function verboseLog(enabled: boolean, message: string): void { + if (!enabled) { + return; + } + + process.stderr.write(`[verbose] ${message}\n`); +} diff --git a/src/index.ts b/src/index.ts index a26f199..f7a9207 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ const program = new Command() .name("decodo") .description("Official CLI for Decodo APIs") .version(readVersion(), "-V, --version", "output the version number") + .option("-v, --verbose", "Print debug logs to stderr") .option( "--token ", "Basic auth token (overrides DECODO_AUTH_TOKEN and saved config)" diff --git a/src/scrape/services/format-scrape-request-log.ts b/src/scrape/services/format-scrape-request-log.ts new file mode 100644 index 0000000..936cad4 --- /dev/null +++ b/src/scrape/services/format-scrape-request-log.ts @@ -0,0 +1,47 @@ +const SENSITIVE_QUERY_PARAM_KEYS = [ + "auth", + "authorization", + "apikey", + "api_key", + "key", + "password", + "secret", + "token", +] as const; + +function isSensitiveQueryParamKey(key: string): boolean { + const normalized = key.toLowerCase(); + + return SENSITIVE_QUERY_PARAM_KEYS.some((sensitive) => + normalized.includes(sensitive) + ); +} + +function sanitizeUrlForLog(value: string): string { + try { + const parsed = new URL(value); + for (const key of parsed.searchParams.keys()) { + if (isSensitiveQueryParamKey(key)) { + parsed.searchParams.set(key, ""); + } + } + return parsed.toString(); + } catch { + return value; + } +} + +export function formatScrapeRequestLog(body: Record): string { + const target = typeof body.target === "string" ? body.target : "unknown"; + const url = typeof body.url === "string" ? sanitizeUrlForLog(body.url) : null; + if (url !== null) { + return `request target=${target} url=${url}`; + } + + const query = typeof body.query === "string" ? body.query : null; + if (query !== null) { + return `request target=${target} query=${query}`; + } + + return `request target=${target}`; +} diff --git a/src/scrape/services/run-target-scrape.ts b/src/scrape/services/run-target-scrape.ts index b77b2ec..11934d6 100644 --- a/src/scrape/services/run-target-scrape.ts +++ b/src/scrape/services/run-target-scrape.ts @@ -1,30 +1,36 @@ import type { DecodoSchema, ScrapeRequest } from "@decodo/sdk-ts"; import type { Command } from "commander"; -import { getRootOpts } from "../../auth/services/global-opts.js"; -import { requireAuthToken } from "../../auth/services/resolve-token.js"; +import { AuthRequiredError } from "../../auth/errors/auth-required-error.js"; +import { resolveAuthToken } from "../../auth/services/resolve-token.js"; +import { getRootOpts } from "../../cli/services/global-opts.js"; +import { verboseLog } from "../../cli/services/verbose-log.js"; import { writeScrapeResponse } from "../../output/services/write-scrape-response.js"; import type { OutputOptions } from "../../output/types/output-options.js"; -import type { WriteScrapeResponseContext } from "../../output/types/write-scrape-response.js"; import { handleCliError } from "../../platform/services/handle-cli-error.js"; import type { + ExecuteScrapeOptions, OutputContextBuilder, ScrapeBodyBuilder, } from "../types/run-target-scrape.js"; import { createDecodoClient } from "./client.js"; import { buildScrapeBody, getTargetCommandConfig } from "./command-builder.js"; +import { formatScrapeRequestLog } from "./format-scrape-request-log.js"; -export async function executeScrape( - token: string, - schema: DecodoSchema, - body: Record, - options: Record, - outputContext?: Partial, - input?: string -): Promise { +async function executeScrape({ + token, + schema, + body, + options, + outputContext, + input, + verbose = false, +}: ExecuteScrapeOptions): Promise { const client = createDecodoClient(token, schema); + const startedAt = Date.now(); const response = await client.webScrapingApi.scrape( body as unknown as ScrapeRequest ); + verboseLog(verbose, `response latency_ms=${Date.now() - startedAt}`); writeScrapeResponse(response, { options: options as OutputOptions, @@ -51,12 +57,27 @@ export function createTargetAction( command: Command ): Promise => { const rootOpts = getRootOpts(command); + const verbose = rootOpts.verbose === true; try { - const token = await requireAuthToken({ token: rootOpts.token }); + const auth = await resolveAuthToken({ token: rootOpts.token }); + verboseLog(verbose, `auth source=${auth.source}`); + if (!auth.token) { + throw new AuthRequiredError(); + } + const body = resolveBody(input, options); + verboseLog(verbose, formatScrapeRequestLog(body)); const outputContext = getOutputContext?.(input, options); - await executeScrape(token, schema, body, options, outputContext, input); + await executeScrape({ + token: auth.token, + schema, + body, + options, + outputContext, + input, + verbose, + }); } catch (err) { handleCliError(err, { fallbackMessage: "Scrape failed." }); } diff --git a/src/scrape/types/run-target-scrape.ts b/src/scrape/types/run-target-scrape.ts index 42ab9da..b974b5f 100644 --- a/src/scrape/types/run-target-scrape.ts +++ b/src/scrape/types/run-target-scrape.ts @@ -1,5 +1,16 @@ +import type { DecodoSchema } from "@decodo/sdk-ts"; import type { WriteScrapeResponseContext } from "../../output/types/write-scrape-response.js"; +export interface ExecuteScrapeOptions { + body: Record; + input?: string; + options: Record; + outputContext?: Partial; + schema: DecodoSchema; + token: string; + verbose?: boolean; +} + export type ScrapeBodyBuilder = ( input: string | undefined, options: Record diff --git a/tests/auth/services/resolve-token.test.ts b/tests/auth/services/resolve-token.test.ts index 75fb605..7f3de9e 100644 --- a/tests/auth/services/resolve-token.test.ts +++ b/tests/auth/services/resolve-token.test.ts @@ -71,36 +71,3 @@ describe("resolveAuthToken", () => { expect(result).toEqual({ token: undefined, source: "none" }); }); }); - -describe("requireAuthToken", () => { - let restoreConfigHome: () => void; - let previousEnvToken: string | undefined; - - beforeEach(async () => { - ({ restore: restoreConfigHome } = await isolateConfigHome()); - previousEnvToken = process.env.DECODO_AUTH_TOKEN; - delete process.env.DECODO_AUTH_TOKEN; - vi.resetModules(); - }); - - afterEach(() => { - restoreConfigHome(); - if (previousEnvToken === undefined) { - delete process.env.DECODO_AUTH_TOKEN; - } else { - process.env.DECODO_AUTH_TOKEN = previousEnvToken; - } - vi.resetModules(); - }); - - it("throws AuthRequiredError when no token is available", async () => { - const { requireAuthToken } = await import( - "../../../src/auth/services/resolve-token.js" - ); - const { AuthRequiredError } = await import( - "../../../src/auth/errors/auth-required-error.js" - ); - await expect(requireAuthToken()).rejects.toBeInstanceOf(AuthRequiredError); - await expect(requireAuthToken()).rejects.toThrow("decodo setup"); - }); -}); diff --git a/tests/index.test.ts b/tests/index.test.ts index 1ff97d4..e90bd01 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -19,4 +19,14 @@ describe("cli", () => { expect(output).toBe(packageJson.version); }); + + it("shows verbose flag in root help", () => { + const output = execFileSync( + process.execPath, + [join(rootDir, "..", "build", "esm", "index.js"), "--help"], + { encoding: "utf8" } + ); + + expect(output).toContain("-v, --verbose"); + }); }); diff --git a/tests/scrape/commands/scrape.test.ts b/tests/scrape/commands/scrape.test.ts index 8cfe7da..9bc9839 100644 --- a/tests/scrape/commands/scrape.test.ts +++ b/tests/scrape/commands/scrape.test.ts @@ -1,12 +1,12 @@ import { BundledSchema, ValidationError } from "@decodo/sdk-ts"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { requireAuthToken } from "../../../src/auth/services/resolve-token.js"; +import { resolveAuthToken } from "../../../src/auth/services/resolve-token.js"; import { createScrapeCommand } from "../../../src/scrape/commands/scrape.js"; import { createDecodoClient } from "../../../src/scrape/services/client.js"; vi.mock("../../../src/auth/services/resolve-token.js", () => ({ - requireAuthToken: vi.fn(), + resolveAuthToken: vi.fn(), })); vi.mock("../../../src/scrape/services/client.js", () => ({ @@ -26,7 +26,10 @@ describe("createScrapeCommand", () => { configurable: true, }); - vi.mocked(requireAuthToken).mockResolvedValue("test-token"); + vi.mocked(resolveAuthToken).mockResolvedValue({ + token: "test-token", + source: "flag", + }); vi.spyOn(process, "exit").mockImplementation((code) => { exitCode = code as number; throw new Error(`process.exit:${code}`); diff --git a/tests/scrape/commands/screenshot.test.ts b/tests/scrape/commands/screenshot.test.ts index 35b74c6..c8a89f0 100644 --- a/tests/scrape/commands/screenshot.test.ts +++ b/tests/scrape/commands/screenshot.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { BundledSchema, ValidationError } from "@decodo/sdk-ts"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { requireAuthToken } from "../../../src/auth/services/resolve-token.js"; +import { resolveAuthToken } from "../../../src/auth/services/resolve-token.js"; import { BINARY_TTY_ERROR } from "../../../src/platform/services/write-binary.js"; import { createScreenshotCommand } from "../../../src/scrape/commands/screenshot.js"; import { createDecodoClient } from "../../../src/scrape/services/client.js"; @@ -13,7 +13,7 @@ const MINIMAL_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAD0JEQVQImWP4GwADAwEAAv6C7p4AAAAASUVORK5CYII="; vi.mock("../../../src/auth/services/resolve-token.js", () => ({ - requireAuthToken: vi.fn(), + resolveAuthToken: vi.fn(), })); vi.mock("../../../src/scrape/services/client.js", () => ({ @@ -30,7 +30,10 @@ describe("createScreenshotCommand", () => { stderr = []; stdoutBytes = undefined; - vi.mocked(requireAuthToken).mockResolvedValue("test-token"); + vi.mocked(resolveAuthToken).mockResolvedValue({ + token: "test-token", + source: "flag", + }); vi.spyOn(process, "exit").mockImplementation((code) => { exitCode = code as number; throw new Error(`process.exit:${code}`); diff --git a/tests/scrape/commands/search.test.ts b/tests/scrape/commands/search.test.ts index 781c565..8bf840e 100644 --- a/tests/scrape/commands/search.test.ts +++ b/tests/scrape/commands/search.test.ts @@ -1,12 +1,12 @@ import { BundledSchema, ValidationError } from "@decodo/sdk-ts"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { requireAuthToken } from "../../../src/auth/services/resolve-token.js"; +import { resolveAuthToken } from "../../../src/auth/services/resolve-token.js"; import { createSearchCommand } from "../../../src/scrape/commands/search.js"; import { createDecodoClient } from "../../../src/scrape/services/client.js"; vi.mock("../../../src/auth/services/resolve-token.js", () => ({ - requireAuthToken: vi.fn(), + resolveAuthToken: vi.fn(), })); vi.mock("../../../src/scrape/services/client.js", () => ({ @@ -26,7 +26,10 @@ describe("createSearchCommand", () => { configurable: true, }); - vi.mocked(requireAuthToken).mockResolvedValue("test-token"); + vi.mocked(resolveAuthToken).mockResolvedValue({ + token: "test-token", + source: "flag", + }); vi.spyOn(process, "exit").mockImplementation((code) => { exitCode = code as number; throw new Error(`process.exit:${code}`); diff --git a/tests/scrape/services/format-scrape-request-log.test.ts b/tests/scrape/services/format-scrape-request-log.test.ts new file mode 100644 index 0000000..011ff2f --- /dev/null +++ b/tests/scrape/services/format-scrape-request-log.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { formatScrapeRequestLog } from "../../../src/scrape/services/format-scrape-request-log.js"; + +describe("formatScrapeRequestLog", () => { + it("formats query-based requests", () => { + expect( + formatScrapeRequestLog({ target: "google_search", query: "coffee" }) + ).toBe("request target=google_search query=coffee"); + }); + + it("redacts sensitive URL query params", () => { + expect( + formatScrapeRequestLog({ + target: "universal", + url: "https://example.com?token=secret&page=1", + }) + ).toBe( + "request target=universal url=https://example.com/?token=%3Credacted%3E&page=1" + ); + }); + + it("redacts compound sensitive URL query params", () => { + expect( + formatScrapeRequestLog({ + target: "universal", + url: "https://example.com?access_token=secret&page=1", + }) + ).toBe( + "request target=universal url=https://example.com/?access_token=%3Credacted%3E&page=1" + ); + }); + + it("falls back to target only when body has no url or query", () => { + expect(formatScrapeRequestLog({ target: "amazon_product" })).toBe( + "request target=amazon_product" + ); + }); +}); diff --git a/tests/scrape/services/run-target-scrape.test.ts b/tests/scrape/services/run-target-scrape.test.ts index ec4018a..0e64764 100644 --- a/tests/scrape/services/run-target-scrape.test.ts +++ b/tests/scrape/services/run-target-scrape.test.ts @@ -1,33 +1,40 @@ import { BundledSchema, ValidationError } from "@decodo/sdk-ts"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { requireAuthToken } from "../../../src/auth/services/resolve-token.js"; +import { resolveAuthToken } from "../../../src/auth/services/resolve-token.js"; import { attachScrapeOutputOptions } from "../../../src/output/commands/attach-output-options.js"; import { createDecodoClient } from "../../../src/scrape/services/client.js"; import { createTargetAction } from "../../../src/scrape/services/run-target-scrape.js"; vi.mock("../../../src/auth/services/resolve-token.js", () => ({ - requireAuthToken: vi.fn(), + resolveAuthToken: vi.fn(), })); vi.mock("../../../src/scrape/services/client.js", () => ({ createDecodoClient: vi.fn(), })); +const RESPONSE_LATENCY_LOG_PATTERN = /\[verbose\] response latency_ms=\d+\n/; + describe("createTargetAction", () => { let exitCode: number | undefined; let stdout: string | undefined; + let stderr: string | undefined; beforeEach(() => { exitCode = undefined; stdout = undefined; + stderr = undefined; Object.defineProperty(process.stdout, "isTTY", { value: false, configurable: true, }); - vi.mocked(requireAuthToken).mockResolvedValue("test-token"); + vi.mocked(resolveAuthToken).mockResolvedValue({ + token: "test-token", + source: "flag", + }); vi.spyOn(process, "exit").mockImplementation((code) => { exitCode = code as number; throw new Error(`process.exit:${code}`); @@ -36,6 +43,10 @@ describe("createTargetAction", () => { stdout = String(chunk); return true; }); + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + stderr = (stderr ?? "") + String(chunk); + return true; + }); vi.spyOn(console, "error").mockImplementation(vi.fn()); }); @@ -71,11 +82,69 @@ describe("createTargetAction", () => { parse: true, markdown: false, }); + expect(createDecodoClient).toHaveBeenCalledWith( + "test-token", + BundledSchema.shared + ); expect(stdout).toBe('{"ok":true}\n'); }); - it("maps requireAuthToken failures through handleCliError", async () => { - vi.mocked(requireAuthToken).mockRejectedValue( + it("prints verbose logs to stderr when --verbose is set", async () => { + const scrape = vi.fn().mockResolvedValue({ + results: [{ content: { ok: true } }], + }); + vi.mocked(createDecodoClient).mockReturnValue({ + webScrapingApi: { scrape }, + } as never); + + const program = new Command() + .option("--token ") + .option("-v, --verbose") + .addCommand( + new Command("google-search") + .argument("") + .action(createTargetAction("google_search", BundledSchema.shared)) + ); + + await program.parseAsync( + ["google-search", "coffee", "--token", "test-token", "--verbose"], + { from: "user" } + ); + + expect(stderr).toContain("[verbose] auth source=flag\n"); + expect(stderr).toContain( + "[verbose] request target=google_search query=coffee\n" + ); + expect(stderr).toMatch(RESPONSE_LATENCY_LOG_PATTERN); + expect(stderr).not.toContain("test-token"); + }); + + it("keeps stderr silent when --verbose is not set", async () => { + const scrape = vi.fn().mockResolvedValue({ + results: [{ content: { ok: true } }], + }); + vi.mocked(createDecodoClient).mockReturnValue({ + webScrapingApi: { scrape }, + } as never); + + const program = new Command() + .option("--token ") + .addCommand( + new Command("google-search") + .argument("") + .action(createTargetAction("google_search", BundledSchema.shared)) + ); + + await program.parseAsync( + ["google-search", "coffee", "--token", "test-token"], + { from: "user" } + ); + + expect(stderr).toBeUndefined(); + }); + + it("maps resolveAuthToken failures through handleCliError", async () => { + vi.mocked(resolveAuthToken).mockRejectedValue( new SyntaxError("Unexpected token in config.json") );