Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
437 changes: 437 additions & 0 deletions docs/cli.md

Large diffs are not rendered by default.

11 changes: 10 additions & 1 deletion src/cli/commands/analyze.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,17 @@ export function registerAnalyzeCommand(program: Command): void {
program
.command("analyze")
.description("Run Stage 1: Plugin Analysis only")
.optionsGroup("Input Options:")
.option("-p, --plugin <path>", "Path to plugin directory")
.option("-c, --config <path>", "Path to config file")
.option("-c, --config <path>", "Path to config file (default: config.yaml)")
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval analyze -p ./my-plugin
$ cc-plugin-eval analyze -p ./my-plugin -c custom-config.yaml
`,
)
.action(async (options: Record<string, unknown>) => {
try {
const cliOptions = extractCLIOptions(options);
Expand Down
11 changes: 10 additions & 1 deletion src/cli/commands/execute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,18 @@ export function registerExecuteCommand(program: Command): void {
program
.command("execute")
.description("Run Stages 1-3: Analysis, Generation, and Execution")
.optionsGroup("Input Options:")
.option("-p, --plugin <path>", "Path to plugin directory")
.option("-c, --config <path>", "Path to config file")
.option("-c, --config <path>", "Path to config file (default: config.yaml)")
.option("--verbose", "Detailed progress output")
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval execute -p ./my-plugin
$ cc-plugin-eval execute -p ./my-plugin --verbose
`,
)
.action(async (options: Record<string, unknown>) => {
try {
const cliOptions = extractCLIOptions(options);
Expand Down
14 changes: 12 additions & 2 deletions src/cli/commands/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,20 @@ export function registerGenerateCommand(program: Command): void {
program
.command("generate")
.description("Run Stages 1-2: Analysis and Scenario Generation")
.optionsGroup("Input Options:")
.option("-p, --plugin <path>", "Path to plugin directory")
.option("-c, --config <path>", "Path to config file")
.option("-c, --config <path>", "Path to config file (default: config.yaml)")
.optionsGroup("Testing Options:")
.option("--verbose", "Detailed progress output")
.option("--semantic", "Enable semantic variation testing")
.option("--semantic", "Generate prompt variations to test robustness")
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval generate -p ./my-plugin
$ cc-plugin-eval generate -p ./my-plugin --semantic --verbose
`,
)
.action(async (options: Record<string, unknown>) => {
try {
const cliOptions = extractCLIOptions(options);
Expand Down
11 changes: 10 additions & 1 deletion src/cli/commands/list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,16 @@ export function registerListCommand(program: Command): void {
program
.command("list")
.description("List previous runs")
.option("-p, --plugin <name>", "Plugin name")
.optionsGroup("Filter Options:")
.option("-p, --plugin <name>", "Filter by plugin name")
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval list
$ cc-plugin-eval list -p my-plugin
`,
)
.action((options: Record<string, unknown>) => {
const pluginName = options["plugin"] as string | undefined;

Expand Down
12 changes: 12 additions & 0 deletions src/cli/commands/report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,22 @@ export function registerReportCommand(program: Command): void {
program
.command("report")
.description("Generate report from existing results")
.optionsGroup("Identification Options:")
.option("-r, --run-id <id>", "Run ID to report on")
.option("-p, --plugin <name>", "Plugin name")
.optionsGroup("Output Options:")
.option("-o, --output <format>", "Output format: json|yaml|junit-xml|tap")
.option("--cli", "Output CLI summary")
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval report -p my-plugin
$ cc-plugin-eval report -r abc123 -o yaml
$ cc-plugin-eval report -r abc123 -o junit-xml > results.xml
$ cc-plugin-eval report -r abc123 --cli
`,
)
.action((options: Record<string, unknown>) => {
try {
// Extract and validate CLI options
Expand Down
12 changes: 11 additions & 1 deletion src/cli/commands/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,12 +307,22 @@ export function registerResumeCommand(program: Command): void {
program
.command("resume")
.description("Resume from saved state")
.optionsGroup("Identification Options:")
.option("-r, --run-id <id>", "Run ID to resume")
.option("-p, --plugin <name>", "Plugin name (for finding run)")
.option("-p, --plugin <name>", "Plugin name (finds latest run)")
.option(
"-s, --from-stage <stage>",
"Stage to resume from: analysis|generation|execution|evaluation",
)
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval resume -p my-plugin
$ cc-plugin-eval resume -r abc123
$ cc-plugin-eval resume -r abc123 -s execution
`,
)
.action(async (options: Record<string, unknown>) => {
try {
const extracted = extractResumeOptions(options);
Expand Down
21 changes: 18 additions & 3 deletions src/cli/commands/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,28 @@ export function registerRunCommand(program: Command): void {
"--with-plugins <paths>",
"Additional plugins for conflict testing (comma-separated)",
)
.option("--semantic", "Enable semantic variation testing")
.option("--semantic", "Generate prompt variations to test robustness")
.option(
"--samples <n>",
"Number of samples for multi-sample judgment",
"Multi-sample judgment count (improves confidence)",
parseInt,
)
.option("--reps <n>", "Number of repetitions per scenario", parseInt)
.option(
"--reps <n>",
"Repeat each scenario N times (measures variance)",
parseInt,
)
.addHelpText(
"after",
`
Examples:
$ cc-plugin-eval run -p ./my-plugin
$ cc-plugin-eval run -p ./my-plugin --dry-run
$ cc-plugin-eval run -p ./my-plugin --estimate
$ cc-plugin-eval run -p ./my-plugin --fast --failed-run abc123
$ cc-plugin-eval run -p ./my-plugin --semantic --reps 3
`,
)
.action(async (options: Record<string, unknown>) => {
try {
const cliOptions = extractCLIOptions(options);
Expand Down