diff --git a/src/infra/config.ts b/src/infra/config.ts index f555326..20f0ad0 100644 --- a/src/infra/config.ts +++ b/src/infra/config.ts @@ -46,6 +46,8 @@ function createDefaultConfig(): AppConfig { apiHeaders: {}, reasoningEffort: DEFAULT_LLM_REASONING_EFFORT, stream: false, + showInteraction: false, + interactionMode: 'summary', activeProfile: '', profiles: {}, }, @@ -187,6 +189,8 @@ export class ConfigService { }, reasoningEffort: this.normalizeReasoningEffort(config.llm?.reasoningEffort), stream: config.llm?.stream === true, + showInteraction: config.llm?.showInteraction === true, + interactionMode: this.normalizeInteractionMode(config.llm?.interactionMode), profiles: this.normalizeProviderProfiles(config.llm?.profiles), }, automation: { @@ -234,6 +238,10 @@ export class ConfigService { } } + private normalizeInteractionMode(value: unknown): AppConfig['llm']['interactionMode'] { + return value === 'raw' ? 'raw' : 'summary'; + } + private normalizeProviderProfiles( profiles: AppConfig['llm']['profiles'] = {}, ): AppConfig['llm']['profiles'] { diff --git a/src/infra/index.ts b/src/infra/index.ts index c55ee11..d02d43a 100644 --- a/src/infra/index.ts +++ b/src/infra/index.ts @@ -14,6 +14,16 @@ export type { } from './github-repo.js'; export { UserCancelledError, isPromptAbortError, isUserCancelledError, getErrorMessage } from './errors.js'; export { DEFAULT_LLM_REASONING_EFFORT, LLM_REASONING_EFFORTS, parseLLMReasoningEffort } from './llm-reasoning.js'; +export { + BufferedLLMInteractionReporter, + NoopLLMInteractionReporter, + createLLMInteractionReporter, +} from './llm-interaction-reporter.js'; +export type { + LLMInteractionEvent, + LLMInteractionReporter, + LLMInteractionStage, +} from './llm-interaction-reporter.js'; export { prompt } from './prompts.js'; export { selectPrompt } from './select.js'; export { ui } from './ui.js'; diff --git a/src/infra/llm-interaction-reporter.ts b/src/infra/llm-interaction-reporter.ts new file mode 100644 index 0000000..dc4316e --- /dev/null +++ b/src/infra/llm-interaction-reporter.ts @@ -0,0 +1,245 @@ +import type { LLMInteractionMode, LLMProvider } from '../types/index.js'; + +export type LLMInteractionStage = + | 'validate' + | 'issue_scoring' + | 'repository_analysis' + | 'patch_draft' + | 'implementation_draft' + | 'validation_repair' + | 'pull_request_draft' + | 'daily_report' + | 'daily_diary'; + +export interface LLMInteractionEvent { + stage: LLMInteractionStage; + model: string; + provider: LLMProvider; + streaming: boolean; + promptChars: number; + context?: string; +} + +export interface LLMInteractionReporter { + onRequestStart(event: LLMInteractionEvent): void; + onResponseChunk(chunk: string): void; + onResponseComplete(event: LLMInteractionEvent & { responseChars: number }): void; + onParseComplete(event: LLMInteractionEvent & { kind: string; status: string; parsed?: unknown }): void; + onRepairStart(event: LLMInteractionEvent & { error: string }): void; +} + +export interface BufferedLLMInteractionReporterOptions { + write?: (line: string) => void; + chunkFlushChars?: number; + mode?: LLMInteractionMode; +} + +export class NoopLLMInteractionReporter implements LLMInteractionReporter { + onRequestStart(): void {} + onResponseChunk(): void {} + onResponseComplete(): void {} + onParseComplete(): void {} + onRepairStart(): void {} +} + +export class BufferedLLMInteractionReporter implements LLMInteractionReporter { + private readonly write: (line: string) => void; + private readonly chunkFlushChars: number; + private readonly mode: LLMInteractionMode; + private chunkBuffer = ''; + private wroteAssistantHeader = false; + + constructor(options: BufferedLLMInteractionReporterOptions = {}) { + this.write = options.write ?? ((line) => process.stdout.write(`${line}\n`)); + this.chunkFlushChars = options.chunkFlushChars ?? 500; + this.mode = options.mode ?? 'summary'; + } + + onRequestStart(event: LLMInteractionEvent): void { + this.chunkBuffer = ''; + this.wroteAssistantHeader = false; + this.write(''); + this.write(`LLM Interaction: ${event.stage}`); + this.write(`Model: ${event.model}`); + this.write(`Provider: ${event.provider}`); + this.write(`Streaming: ${event.streaming ? 'yes' : 'no'}`); + this.write(`Prompt: ${event.promptChars.toLocaleString()} chars`); + if (event.context) { + this.write(`Context: ${event.context}`); + } + } + + onResponseChunk(chunk: string): void { + if (this.mode !== 'raw') { + return; + } + + if (!chunk) { + return; + } + + this.chunkBuffer += chunk; + if (this.chunkBuffer.length >= this.chunkFlushChars) { + this.flushChunks(); + } + } + + onResponseComplete(event: LLMInteractionEvent & { responseChars: number }): void { + this.flushChunks(); + this.write(`Assistant response received: ${event.responseChars.toLocaleString()} chars`); + } + + onParseComplete(event: LLMInteractionEvent & { kind: string; status: string; parsed?: unknown }): void { + this.write(`Parsed: ${event.kind} / ${event.status}`); + if (this.mode !== 'summary') { + return; + } + + for (const line of this.describeParsedOutput(event.kind, event.parsed)) { + this.write(line); + } + } + + onRepairStart(event: LLMInteractionEvent & { error: string }): void { + this.write(`Structured output parse failed; requesting repair. ${event.error}`); + } + + private flushChunks(): void { + if (!this.chunkBuffer) { + return; + } + + if (!this.wroteAssistantHeader) { + this.write('Assistant:'); + this.wroteAssistantHeader = true; + } + + this.write(this.chunkBuffer); + this.chunkBuffer = ''; + } + + private describeParsedOutput(kind: string, parsed: unknown): string[] { + const data = this.getEnvelopeData(parsed); + + if (kind === 'repository_suggestion_list' && Array.isArray(data)) { + return [ + `Repository suggestions: ${data.length}`, + ...data.slice(0, 5).map((item) => { + const record = this.asRecord(item); + const title = this.asString(record?.['title'], 'Untitled suggestion'); + const score = this.asNumber(record?.['prPotentialScore']); + const files = this.formatTargetFiles(record?.['targetFiles']); + return `- ${title}${score === undefined ? '' : ` | score ${score}`}${files ? ` | files ${files}` : ''}`; + }), + ]; + } + + if (kind === 'patch_draft' && data) { + const record = this.asRecord(data); + return [ + `Patch goal: ${this.asString(record?.['goal'], 'Not provided')}`, + `Target files: ${this.formatTargetFiles(record?.['targetFiles']) || 'none'}`, + `Changes: ${this.countArray(record?.['proposedChanges'])}`, + `Risks: ${this.countArray(record?.['risks'])}`, + ]; + } + + if (kind === 'implementation_draft' && data) { + const record = this.asRecord(data); + return [ + `Implementation summary: ${this.asString(record?.['summary'], 'Not provided')}`, + `File changes: ${this.formatFileChanges(record?.['fileChanges']) || 'none'}`, + ]; + } + + if (kind === 'pull_request_draft' && data) { + const record = this.asRecord(data); + return [ + `PR title: ${this.asString(record?.['title'], 'Not provided')}`, + `Changes: ${this.countArray(record?.['changes'])}`, + `Validation notes: ${this.countArray(record?.['validation'])}`, + `Risks: ${this.countArray(record?.['risks'])}`, + ]; + } + + if (kind === 'issue_match_list' && Array.isArray(data)) { + return [ + `Matched issues: ${data.length}`, + ...data.slice(0, 5).map((item) => { + const record = this.asRecord(item); + const reference = `${this.asString(record?.['repoFullName'], 'unknown')}#${this.asString(record?.['number'], '?')}`; + const title = this.asString(record?.['title'], 'Untitled issue'); + const score = this.asNumber(record?.['matchScore']); + return `- ${reference} | ${title}${score === undefined ? '' : ` | score ${score}`}`; + }), + ]; + } + + return []; + } + + private getEnvelopeData(value: unknown): unknown { + const record = this.asRecord(value); + return record?.['data']; + } + + private formatTargetFiles(value: unknown): string { + if (!Array.isArray(value)) { + return ''; + } + return value + .slice(0, 5) + .map((item) => this.asString(this.asRecord(item)?.['path'], '')) + .filter(Boolean) + .join(', '); + } + + private formatFileChanges(value: unknown): string { + if (!Array.isArray(value)) { + return ''; + } + return value + .slice(0, 5) + .map((item) => this.asString(this.asRecord(item)?.['path'], '')) + .filter(Boolean) + .join(', '); + } + + private countArray(value: unknown): string { + return Array.isArray(value) ? String(value.length) : '0'; + } + + private asRecord(value: unknown): Record | undefined { + return typeof value === 'object' && value !== null ? value as Record : undefined; + } + + private asString(value: unknown, fallback: string): string { + if (typeof value === 'string') { + return value; + } + if (typeof value === 'number') { + return String(value); + } + return fallback; + } + + private asNumber(value: unknown): number | undefined { + return typeof value === 'number' ? value : undefined; + } +} + +export function createLLMInteractionReporter( + enabled: boolean, + modeOrOptions?: LLMInteractionMode | BufferedLLMInteractionReporterOptions, + options?: BufferedLLMInteractionReporterOptions, +): LLMInteractionReporter { + if (!enabled) { + return new NoopLLMInteractionReporter(); + } + + if (typeof modeOrOptions === 'string') { + return new BufferedLLMInteractionReporter({ ...options, mode: modeOrOptions }); + } + + return new BufferedLLMInteractionReporter(modeOrOptions); +} diff --git a/src/orchestration/agent.ts b/src/orchestration/agent.ts index 7293428..5708cfd 100644 --- a/src/orchestration/agent.ts +++ b/src/orchestration/agent.ts @@ -14,6 +14,7 @@ import type { } from '../types/index.js'; import { configService, + createLLMInteractionReporter, ensureDirectory, getLocalDateStamp, getOpenMetaArtifactRoot, @@ -808,6 +809,9 @@ export class AgentOrchestrator { config.llm.provider, config.llm.reasoningEffort, config.llm.stream === true, + config.llm.showInteraction === true, + createLLMInteractionReporter(config.llm.showInteraction === true, config.llm.interactionMode || 'summary'), + config.llm.interactionMode || 'summary', ); const llmValid = await ui.task({ title: 'Validating LLM provider', diff --git a/src/orchestration/analyze.ts b/src/orchestration/analyze.ts index 251b068..19a3584 100644 --- a/src/orchestration/analyze.ts +++ b/src/orchestration/analyze.ts @@ -4,6 +4,7 @@ import type { PatchDraft, PullRequestDraft, RepositoryImprovementSuggestion } fr import type { AppConfig, RankedIssue, RepoWorkspaceContext } from '../types/index.js'; import { configService, + createLLMInteractionReporter, ensureDirectory, getLocalDateStamp, getOpenMetaArtifactRoot, @@ -204,6 +205,9 @@ export class AnalyzeOrchestrator { config.llm.provider, config.llm.reasoningEffort, config.llm.stream === true, + config.llm.showInteraction === true, + createLLMInteractionReporter(config.llm.showInteraction === true, config.llm.interactionMode || 'summary'), + config.llm.interactionMode || 'summary', ); const llmValid = await ui.task({ title: 'Validating LLM provider', diff --git a/src/orchestration/config.ts b/src/orchestration/config.ts index 782abaf..356fbad 100644 --- a/src/orchestration/config.ts +++ b/src/orchestration/config.ts @@ -64,6 +64,8 @@ export class ConfigOrchestrator { { label: 'Model', value: config.llm.modelName || '(not set)', tone: config.llm.modelName ? 'info' : 'warning' }, { label: 'Reasoning effort', value: config.llm.reasoningEffort || 'none', tone: 'info' }, { label: 'Streaming', value: config.llm.stream ? 'yes' : 'no', tone: config.llm.stream ? 'info' : 'muted' }, + { label: 'LLM interaction output', value: config.llm.showInteraction ? 'yes' : 'no', tone: config.llm.showInteraction ? 'info' : 'muted' }, + { label: 'LLM interaction mode', value: config.llm.interactionMode || 'summary', tone: config.llm.showInteraction ? 'info' : 'muted' }, { label: 'Extra headers', value: Object.keys(config.llm.apiHeaders || {}).length > 0 ? JSON.stringify(config.llm.apiHeaders) : '(none)', tone: 'info' }, { label: 'API key', value: ui.maskSecret(config.llm.apiKey), tone: config.llm.apiKey ? 'info' : 'warning' }, { label: 'Saved profiles', value: String(Object.keys(config.llm.profiles || {}).length), tone: Object.keys(config.llm.profiles || {}).length > 0 ? 'info' : 'muted' }, @@ -99,7 +101,7 @@ export class ConfigOrchestrator { async set(key: string, value: string): Promise { const config = await configService.get(); const validPaths = ['userProfile.techStack', 'userProfile.proficiency', 'userProfile.focusAreas', - 'github.username', 'github.pat', 'github.targetRepoPath', 'llm.provider', 'llm.apiBaseUrl', 'llm.apiKey', 'llm.modelName', 'llm.reasoningEffort', 'llm.stream', + 'github.username', 'github.pat', 'github.targetRepoPath', 'llm.provider', 'llm.apiBaseUrl', 'llm.apiKey', 'llm.modelName', 'llm.reasoningEffort', 'llm.stream', 'llm.showInteraction', 'llm.interactionMode', 'automation.enabled', 'automation.scheduleTime', 'automation.contentType', 'automation.minMatchScore', 'automation.skipIfAlreadyGeneratedToday', 'commitTemplate']; @@ -150,6 +152,10 @@ export class ConfigOrchestrator { updated = await configService.update({ llm: { ...config.llm, reasoningEffort: parseLLMReasoningEffort(value) } }); } else if (key === 'llm.stream') { updated = await configService.update({ llm: { ...config.llm, stream: this.parseBoolean(value, key) } }); + } else if (key === 'llm.showInteraction') { + updated = await configService.update({ llm: { ...config.llm, showInteraction: this.parseBoolean(value, key) } }); + } else if (key === 'llm.interactionMode') { + updated = await configService.update({ llm: { ...config.llm, interactionMode: this.parseInteractionMode(value) } }); } else if (key === 'automation.enabled') { updated = await configService.update({ automation: { @@ -312,6 +318,10 @@ export class ConfigOrchestrator { return config.llm.reasoningEffort || 'none'; case 'llm.stream': return config.llm.stream ? 'yes' : 'no'; + case 'llm.showInteraction': + return config.llm.showInteraction ? 'yes' : 'no'; + case 'llm.interactionMode': + return config.llm.interactionMode || 'summary'; case 'automation.enabled': return config.automation.enabled ? 'yes' : 'no'; case 'automation.scheduleTime': @@ -328,6 +338,14 @@ export class ConfigOrchestrator { return '(updated)'; } } + + private parseInteractionMode(value: string): AppConfig['llm']['interactionMode'] { + const normalized = value.trim().toLowerCase(); + if (normalized === 'summary' || normalized === 'raw') { + return normalized; + } + throw new Error('llm.interactionMode must be "summary" or "raw".'); + } } export const configOrchestrator = new ConfigOrchestrator(); diff --git a/src/orchestration/doctor.ts b/src/orchestration/doctor.ts index cb44437..3d81d17 100644 --- a/src/orchestration/doctor.ts +++ b/src/orchestration/doctor.ts @@ -276,7 +276,7 @@ export class DoctorOrchestrator { label: 'LLM configuration', status: 'pass', summary: `${config.llm.provider} provider is configured.`, - detail: `${config.llm.modelName} at ${config.llm.apiBaseUrl}; reasoning ${config.llm.reasoningEffort || 'none'}; streaming ${config.llm.stream ? 'yes' : 'no'}; key ${ui.maskSecret(config.llm.apiKey)}`, + detail: `${config.llm.modelName} at ${config.llm.apiBaseUrl}; reasoning ${config.llm.reasoningEffort || 'none'}; streaming ${config.llm.stream ? 'yes' : 'no'}; interaction output ${config.llm.showInteraction ? `yes (${config.llm.interactionMode || 'summary'})` : 'no'}; key ${ui.maskSecret(config.llm.apiKey)}`, }; } diff --git a/src/orchestration/init.ts b/src/orchestration/init.ts index 1964926..d74a794 100644 --- a/src/orchestration/init.ts +++ b/src/orchestration/init.ts @@ -9,7 +9,7 @@ import { findLLMProviderPreset, type SchedulerSyncResult, } from '../services/index.js'; -import { configService, DEFAULT_LLM_REASONING_EFFORT, LLM_REASONING_EFFORTS, prompt, selectPrompt, ui } from '../infra/index.js'; +import { configService, createLLMInteractionReporter, DEFAULT_LLM_REASONING_EFFORT, LLM_REASONING_EFFORTS, prompt, selectPrompt, ui } from '../infra/index.js'; import type { ContentType } from '../types/content.types.js'; import type { LLMReasoningEffort } from '../types/index.js'; @@ -189,18 +189,21 @@ export class InitOrchestrator { let apiKey = config.llm.apiKey; let reasoningEffort = config.llm.reasoningEffort || DEFAULT_LLM_REASONING_EFFORT; let stream = config.llm.stream === true; + let showInteraction = config.llm.showInteraction === true; + let interactionMode = config.llm.interactionMode || 'summary'; await stepOrSkip( 'llm', !!(apiKey && apiBaseUrl && modelValue), 'LLM provider is already configured.', () => { - llmService.initialize(apiKey, apiBaseUrl, modelValue, apiHeaders, providerValue, reasoningEffort, stream); + llmService.initialize(apiKey, apiBaseUrl, modelValue, apiHeaders, providerValue, reasoningEffort, stream, showInteraction, createLLMInteractionReporter(showInteraction, interactionMode), interactionMode); ui.keyValues('LLM provider connected', [ { label: 'Provider', value: selectedProvider?.name ?? providerValue, tone: 'success' }, { label: 'Model', value: modelValue, tone: 'success' }, { label: 'Reasoning effort', value: reasoningEffort, tone: 'info' }, { label: 'Streaming', value: stream ? 'yes' : 'no', tone: stream ? 'info' : 'muted' }, + { label: 'Interaction output', value: showInteraction ? 'yes' : 'no', tone: showInteraction ? 'info' : 'muted' }, { label: 'Endpoint', value: apiBaseUrl, tone: 'info' }, { label: 'Extra headers', value: Object.keys(apiHeaders).length > 0 ? JSON.stringify(apiHeaders) : '(none)', tone: 'info' }, { label: 'API key', value: ui.maskSecret(apiKey), tone: 'success' }, @@ -238,9 +241,10 @@ export class InitOrchestrator { reasoningEffort = await this.promptReasoningEffort(config.llm.reasoningEffort); stream = await this.promptLlmStreaming(config.llm.stream); + showInteraction = await this.promptLlmInteractionOutput(config.llm.showInteraction); apiKey = await this.promptAPIKey(); - llmService.initialize(apiKey, apiBaseUrl, modelValue, apiHeaders, selectedProvider.value as AppConfig['llm']['provider'], reasoningEffort, stream); + llmService.initialize(apiKey, apiBaseUrl, modelValue, apiHeaders, selectedProvider.value as AppConfig['llm']['provider'], reasoningEffort, stream, showInteraction, createLLMInteractionReporter(showInteraction, interactionMode), interactionMode); llmValid = await this.validateLlmConnection(); if (!llmValid) { @@ -267,12 +271,13 @@ export class InitOrchestrator { } } completedSteps.add('llm'); - await commit({ llm: { provider: providerValue as AppConfig['llm']['provider'], apiBaseUrl, apiKey, modelName: modelValue, apiHeaders, reasoningEffort, stream } }); + await commit({ llm: { provider: providerValue as AppConfig['llm']['provider'], apiBaseUrl, apiKey, modelName: modelValue, apiHeaders, reasoningEffort, stream, showInteraction, interactionMode } }); ui.keyValues('LLM provider connected', [ { label: 'Provider', value: selectedProvider!.name, tone: 'success' }, { label: 'Model', value: modelValue, tone: 'success' }, { label: 'Reasoning effort', value: reasoningEffort, tone: 'info' }, { label: 'Streaming', value: stream ? 'yes' : 'no', tone: stream ? 'info' : 'muted' }, + { label: 'Interaction output', value: showInteraction ? 'yes' : 'no', tone: showInteraction ? 'info' : 'muted' }, { label: 'Endpoint', value: apiBaseUrl, tone: 'info' }, { label: 'Extra headers', value: Object.keys(apiHeaders).length > 0 ? JSON.stringify(apiHeaders) : '(none)', tone: 'info' }, { label: 'API key', value: ui.maskSecret(apiKey), tone: 'success' }, @@ -443,6 +448,7 @@ export class InitOrchestrator { { label: 'Model', value: modelValue, hint: selectedProvider?.name, tone: 'success' }, { label: 'Reasoning', value: reasoningEffort, tone: 'info' }, { label: 'Streaming', value: stream ? 'YES' : 'NO', tone: stream ? 'info' : 'muted' }, + { label: 'LLM output', value: showInteraction ? 'DETAILED' : 'QUIET', tone: showInteraction ? 'info' : 'muted' }, { label: 'Repo policy', value: targetRepoPath ? 'CUSTOM' : 'MANAGED', tone: 'accent' }, { label: 'Automation', value: automationEnabled ? 'ENABLED' : 'MANUAL', tone: automationEnabled ? 'warning' : 'muted' }, ]); @@ -535,6 +541,19 @@ export class InitOrchestrator { return stream; } + private async promptLlmInteractionOutput(defaultValue?: boolean): Promise { + const { showInteraction } = await prompt<{ showInteraction: boolean }>([ + { + type: 'confirm', + name: 'showInteraction', + message: 'Show detailed LLM interaction output?', + default: defaultValue === true, + }, + ]); + + return showInteraction; + } + private async promptUsername(): Promise { const { username } = await prompt<{ username: string }>([ { diff --git a/src/orchestration/provider.ts b/src/orchestration/provider.ts index a0ee0f6..5bf7843 100644 --- a/src/orchestration/provider.ts +++ b/src/orchestration/provider.ts @@ -1,4 +1,4 @@ -import { configService, DEFAULT_LLM_REASONING_EFFORT, LLM_REASONING_EFFORTS, parseLLMReasoningEffort, prompt, selectPrompt, ui } from '../infra/index.js'; +import { configService, createLLMInteractionReporter, DEFAULT_LLM_REASONING_EFFORT, LLM_REASONING_EFFORTS, parseLLMReasoningEffort, prompt, selectPrompt, ui } from '../infra/index.js'; import { LLM_PROVIDER_PRESETS, findLLMProviderPreset } from '../services/index.js'; import { llmService } from '../services/index.js'; import type { AppConfig, LLMProvider, LLMProviderProfile, LLMReasoningEffort } from '../types/index.js'; @@ -376,7 +376,11 @@ export class ProviderOrchestrator { let validationDetail = 'Validation skipped.'; let tone: 'success' | 'warning' = 'success'; if (options.validate) { - const valid = await this.validateProfile(profile); + const valid = await this.validateProfile( + profile, + updated.llm.showInteraction === true, + updated.llm.interactionMode || 'summary', + ); validationDetail = valid ? 'Provider validation succeeded.' : `Provider validation failed: ${llmService.getLastValidationError() || 'unknown reason'}`; @@ -499,7 +503,11 @@ export class ProviderOrchestrator { }); } - private async validateProfile(profile: LLMProviderProfile): Promise { + private async validateProfile( + profile: LLMProviderProfile, + showInteraction: boolean, + interactionMode: NonNullable, + ): Promise { llmService.initialize( profile.apiKey, profile.apiBaseUrl, @@ -508,6 +516,9 @@ export class ProviderOrchestrator { profile.provider, profile.reasoningEffort ?? DEFAULT_LLM_REASONING_EFFORT, profile.stream === true, + showInteraction, + createLLMInteractionReporter(showInteraction, interactionMode), + interactionMode, ); return llmService.validateConnection(); diff --git a/src/services/llm.ts b/src/services/llm.ts index 24c1e2e..d1d3c4f 100644 --- a/src/services/llm.ts +++ b/src/services/llm.ts @@ -15,6 +15,7 @@ import type { GitHubIssue, ImplementationDraft, LLMProvider, + LLMInteractionMode, LLMReasoningEffort, MatchedIssue, RankedIssue, @@ -25,6 +26,11 @@ import type { UserProfile, } from '../types/index.js'; import { logger } from '../infra/logger.js'; +import type { + LLMInteractionEvent, + LLMInteractionReporter, + LLMInteractionStage, +} from '../infra/llm-interaction-reporter.js'; import { LLM_VALIDATION_FALLBACK_HINTS, LLM_VALIDATION_PROMPT, @@ -54,6 +60,9 @@ export class LLMService { private provider: LLMProvider = 'openai'; private reasoningEffort: LLMReasoningEffort | undefined; private stream = false; + private showInteraction = false; + private interactionMode: LLMInteractionMode = 'summary'; + private interactionReporter: LLMInteractionReporter | undefined; private lastValidationError: string | null = null; initialize( @@ -64,6 +73,9 @@ export class LLMService { provider?: LLMProvider, reasoningEffort?: LLMReasoningEffort, stream?: boolean, + showInteraction?: boolean, + interactionReporter?: LLMInteractionReporter, + interactionMode?: LLMInteractionMode, ): void { this.client = new OpenAI({ apiKey, @@ -78,6 +90,9 @@ export class LLMService { } this.reasoningEffort = reasoningEffort; this.stream = stream === true; + this.showInteraction = showInteraction === true; + this.interactionMode = interactionMode ?? 'summary'; + this.interactionReporter = interactionReporter; } async validateConnection(): Promise { @@ -91,6 +106,8 @@ export class LLMService { const timeout = setTimeout(() => controller.abort(), LLM_VALIDATION_TIMEOUT_MS); try { + const interactionEvent = this.createInteractionEvent('validate', LLM_VALIDATION_PROMPT); + this.emitRequestStart(interactionEvent); const response = await this.client.chat.completions.create({ model: this.modelName, messages: [{ role: 'user', content: LLM_VALIDATION_PROMPT }], @@ -100,10 +117,12 @@ export class LLMService { }, { signal: controller.signal, }); + const content = await this.extractChatContent(response); + this.emitResponseComplete({ ...interactionEvent, responseChars: content.length }); // 自定义兼容端点最容易把站点页面或其他 200 响应误判为可用,所以这里额外校验返回结构。 if (this.provider === 'custom') { - await this.assertCustomValidationResponse(response); + await this.assertCustomValidationResponse(response, content); } } finally { clearTimeout(timeout); @@ -154,6 +173,8 @@ Repo Stars: ${i.repoStars}` prompt, parser: (content) => this.parseLLMResponse(content, issues), repairPrompt: ISSUE_MATCH_REPAIR_PROMPT, + stage: 'issue_scoring', + context: `${issues.length} issue${issues.length === 1 ? '' : 's'}`, }); } @@ -162,7 +183,7 @@ Repo Stars: ${i.repoStars}` issueAnalysis, }); - return await this.chat(prompt); + return await this.chat(prompt, { stage: 'daily_report' }); } async generateDailyDiary(issueAnalysis: string, userCodeSnippets: string): Promise { @@ -171,7 +192,7 @@ Repo Stars: ${i.repoStars}` userCodeSnippets: userCodeSnippets || 'No code snippets provided.', }); - return await this.chat(prompt); + return await this.chat(prompt, { stage: 'daily_diary' }); } async generatePatchDraft( @@ -203,6 +224,8 @@ Repo Stars: ${i.repoStars}` prompt, parser: this.parsePatchDraft.bind(this), repairPrompt: PATCH_DRAFT_REPAIR_PROMPT, + stage: 'patch_draft', + context: this.getIssueReference(issue), }); } @@ -235,6 +258,8 @@ Repo Stars: ${i.repoStars}` parser: this.parseRepositorySuggestions.bind(this), repairPrompt: REPOSITORY_ANALYSIS_REPAIR_PROMPT, temperature: 0.2, + stage: 'repository_analysis', + context: repoFullName, }); } @@ -258,6 +283,8 @@ Repo Stars: ${i.repoStars}` parser: this.parseImplementationDraft.bind(this), repairPrompt: CODE_CHANGE_REPAIR_PROMPT, temperature: 0.1, + stage: 'implementation_draft', + context: this.getIssueReference(issue), }); } @@ -281,6 +308,8 @@ Repo Stars: ${i.repoStars}` return this.generateStructuredOutput({ prompt, parser: this.parsePullRequestDraft.bind(this), + stage: 'pull_request_draft', + context: this.getIssueReference(issue), }); } @@ -311,15 +340,19 @@ Repo Stars: ${i.repoStars}` parser: this.parseImplementationDraft.bind(this), repairPrompt: CODE_CHANGE_REPAIR_PROMPT, temperature: 0.1, + stage: 'validation_repair', + context: this.getIssueReference(issue), }); } - private async chat(prompt: string, options: { temperature?: number } = {}): Promise { + private async chat(prompt: string, options: { temperature?: number; stage?: LLMInteractionStage; context?: string } = {}): Promise { if (!this.client) { throw new Error('LLM client not initialized'); } try { + const interactionEvent = this.createInteractionEvent(options.stage ?? 'daily_report', prompt, options.context); + this.emitRequestStart(interactionEvent); const response = await this.client.chat.completions.create({ model: this.modelName, messages: [ @@ -331,7 +364,9 @@ Repo Stars: ${i.repoStars}` ...this.getReasoningRequestParams(), }); - return await this.extractChatContent(response); + const content = await this.extractChatContent(response); + this.emitResponseComplete({ ...interactionEvent, responseChars: content.length }); + return content; } catch (error) { logger.debug('LLM chat failed', error); throw new Error('The LLM request failed. Please verify your provider, model, and API key.'); @@ -343,7 +378,11 @@ Repo Stars: ${i.repoStars}` let content = ''; for await (const chunk of response) { - content += this.extractStreamChunkContent(chunk); + const chunkContent = this.extractStreamChunkContent(chunk); + content += chunkContent; + if (chunkContent && this.interactionMode === 'raw') { + this.emitResponseChunk(chunkContent); + } } return content; @@ -400,25 +439,101 @@ Repo Stars: ${i.repoStars}` parser: (content: string) => T; repairPrompt?: string; temperature?: number; + stage: LLMInteractionStage; + context?: string; }): Promise { - const content = await this.chat(input.prompt, { temperature: input.temperature }); + const interactionEvent = this.createInteractionEvent(input.stage, input.prompt, input.context); + const content = await this.chat(input.prompt, { + temperature: input.temperature, + stage: input.stage, + context: input.context, + }); try { - return input.parser(content); + const parsed = input.parser(content); + this.emitParseComplete(interactionEvent, parsed); + return parsed; } catch (error) { if (!input.repairPrompt) { throw error; } logger.debug('Structured output parsing failed, attempting repair', error); + this.emitRepairStart({ + ...interactionEvent, + error: error instanceof Error ? error.message : String(error), + }); const repairedContent = await this.chat(fillPrompt(input.repairPrompt, { invalidResponse: content.slice(0, 12000), - }), { temperature: 0 }); + }), { temperature: 0, stage: input.stage, context: input.context }); - return input.parser(repairedContent); + const parsed = input.parser(repairedContent); + this.emitParseComplete(interactionEvent, parsed); + return parsed; } } + private createInteractionEvent(stage: LLMInteractionStage, prompt: string, context?: string): LLMInteractionEvent { + return { + stage, + model: this.modelName, + provider: this.provider, + streaming: this.stream, + promptChars: prompt.length, + context, + }; + } + + private emitRequestStart(event: LLMInteractionEvent): void { + this.withInteractionReporter((reporter) => reporter.onRequestStart(event)); + } + + private emitResponseChunk(chunk: string): void { + this.withInteractionReporter((reporter) => reporter.onResponseChunk(chunk)); + } + + private emitResponseComplete(event: LLMInteractionEvent & { responseChars: number }): void { + this.withInteractionReporter((reporter) => reporter.onResponseComplete(event)); + } + + private emitParseComplete(event: LLMInteractionEvent, parsed: unknown): void { + const metadata = this.getStructuredOutputMetadata(parsed); + this.withInteractionReporter((reporter) => reporter.onParseComplete({ + ...event, + kind: metadata.kind, + status: metadata.status, + parsed, + })); + } + + private emitRepairStart(event: LLMInteractionEvent & { error: string }): void { + this.withInteractionReporter((reporter) => reporter.onRepairStart(event)); + } + + private withInteractionReporter(callback: (reporter: LLMInteractionReporter) => void): void { + if (!this.showInteraction || !this.interactionReporter) { + return; + } + + try { + callback(this.interactionReporter); + } catch { + // Reporter output is diagnostic only and must not affect LLM execution. + } + } + + private getStructuredOutputMetadata(parsed: unknown): { kind: string; status: string } { + if (typeof parsed === 'object' && parsed !== null) { + const record = parsed as Record; + return { + kind: typeof record['kind'] === 'string' ? record['kind'] : 'structured_output', + status: typeof record['status'] === 'string' ? record['status'] : 'success', + }; + } + + return { kind: 'structured_output', status: 'success' }; + } + private parseLLMResponse( content: string, originalIssues: GitHubIssue[], @@ -672,9 +787,9 @@ Repo Stars: ${i.repoStars}` return error instanceof Error && (error.name === 'AbortError' || error.message.toLowerCase().includes('aborted')); } - private async assertCustomValidationResponse(response: unknown): Promise { + private async assertCustomValidationResponse(response: unknown, streamedContent?: string): Promise { if (this.isAsyncIterable(response)) { - const content = await this.extractChatContent(response); + const content = streamedContent ?? ''; if (content.trim().length === 0) { throw new Error('Custom provider validation response did not include a usable assistant reply.'); } diff --git a/src/types/config.types.ts b/src/types/config.types.ts index 80ecd6d..f6a9616 100644 --- a/src/types/config.types.ts +++ b/src/types/config.types.ts @@ -17,6 +17,7 @@ export interface GitHubConfig { export type LLMProvider = 'openai' | 'minimax' | 'moonshot' | 'zhipu' | 'gemini' | 'claude' | 'custom'; export type LLMReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; +export type LLMInteractionMode = 'summary' | 'raw'; export interface LLMProviderProfile { provider: LLMProvider; @@ -36,6 +37,8 @@ export interface LLMConfig { apiHeaders?: Record; reasoningEffort?: LLMReasoningEffort; stream?: boolean; + showInteraction?: boolean; + interactionMode?: LLMInteractionMode; activeProfile?: string; profiles?: Record; } diff --git a/test/config-orchestrator.test.ts b/test/config-orchestrator.test.ts index f46cef5..95af29e 100644 --- a/test/config-orchestrator.test.ts +++ b/test/config-orchestrator.test.ts @@ -73,4 +73,32 @@ describe('ConfigOrchestrator', () => { 'llm.stream must be a boolean value.', ); }); + + test('sets and validates LLM interaction output from dotted config keys', async () => { + const orchestrator = new ConfigOrchestrator(); + + await orchestrator.set('llm.showInteraction', 'true'); + expect((await configService.get()).llm.showInteraction).toBe(true); + + await orchestrator.set('llm.showInteraction', 'false'); + expect((await configService.get()).llm.showInteraction).toBe(false); + + await expect(orchestrator.set('llm.showInteraction', 'maybe')).rejects.toThrow( + 'llm.showInteraction must be a boolean value.', + ); + }); + + test('sets and validates LLM interaction mode from dotted config keys', async () => { + const orchestrator = new ConfigOrchestrator(); + + await orchestrator.set('llm.interactionMode', 'raw'); + expect((await configService.get()).llm.interactionMode).toBe('raw'); + + await orchestrator.set('llm.interactionMode', 'summary'); + expect((await configService.get()).llm.interactionMode).toBe('summary'); + + await expect(orchestrator.set('llm.interactionMode', 'verbose')).rejects.toThrow( + 'llm.interactionMode must be "summary" or "raw".', + ); + }); }); diff --git a/test/doctor.test.ts b/test/doctor.test.ts index 4d027a9..5415279 100644 --- a/test/doctor.test.ts +++ b/test/doctor.test.ts @@ -93,6 +93,25 @@ describe('DoctorOrchestrator', () => { expect(report.checks.find((check) => check.id === 'llm-config')?.status).toBe('pass'); }); + test('includes LLM interaction output state in diagnostics', async () => { + const report = await new DoctorOrchestrator().inspect(createConfig({ + llm: { + provider: 'openai', + apiBaseUrl: 'https://api.openai.com/v1', + apiKey: 'sk-test-key', + modelName: 'gpt-5.5', + stream: true, + showInteraction: true, + interactionMode: 'summary', + }, + })); + + const detail = report.checks.find((check) => check.id === 'llm-config')?.detail; + + expect(detail).toContain('streaming yes'); + expect(detail).toContain('interaction output yes (summary)'); + }); + test('marks missing credentials as critical failures', async () => { const report = await new DoctorOrchestrator().inspect(createConfig({ github: { diff --git a/test/init-orchestrator.test.ts b/test/init-orchestrator.test.ts index 9582872..e6e0e65 100644 --- a/test/init-orchestrator.test.ts +++ b/test/init-orchestrator.test.ts @@ -6,6 +6,7 @@ import type { LLMReasoningEffort } from '../src/types/index.js'; interface InitOrchestratorInternals { promptReasoningEffort(defaultValue?: LLMReasoningEffort): Promise; promptLlmStreaming(defaultValue?: boolean): Promise; + promptLlmInteractionOutput(defaultValue?: boolean): Promise; } describe('InitOrchestrator LLM reasoning setup', () => { @@ -50,4 +51,25 @@ describe('InitOrchestrator LLM reasoning setup', () => { promptSpy.mockRestore(); } }); + + test('defaults interaction output selection to false during init', async () => { + const orchestrator = new InitOrchestrator() as unknown as InitOrchestratorInternals; + const promptSpy = spyOn(infra, 'prompt').mockResolvedValue({ showInteraction: false }); + + try { + const selected = await orchestrator.promptLlmInteractionOutput(); + + expect(selected).toBe(false); + expect(promptSpy).toHaveBeenCalledWith([ + expect.objectContaining({ + type: 'confirm', + name: 'showInteraction', + message: 'Show detailed LLM interaction output?', + default: false, + }), + ]); + } finally { + promptSpy.mockRestore(); + } + }); }); diff --git a/test/llm-interaction-reporter.test.ts b/test/llm-interaction-reporter.test.ts new file mode 100644 index 0000000..dae079e --- /dev/null +++ b/test/llm-interaction-reporter.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from 'bun:test'; +import { BufferedLLMInteractionReporter, createLLMInteractionReporter } from '../src/infra/llm-interaction-reporter.js'; + +describe('BufferedLLMInteractionReporter', () => { + test('summary mode suppresses raw chunks and renders parsed repository suggestions', () => { + const lines: string[] = []; + const reporter = new BufferedLLMInteractionReporter({ + write: (line) => lines.push(line), + chunkFlushChars: 10, + mode: 'summary', + }); + + reporter.onRequestStart({ + stage: 'repository_analysis', + model: 'gpt-5.5', + provider: 'custom', + streaming: true, + promptChars: 1200, + context: 'acme/demo', + }); + reporter.onResponseChunk('{"version":"1","kind":"repository_suggestion_list"'); + reporter.onResponseComplete({ + stage: 'repository_analysis', + model: 'gpt-5.5', + provider: 'custom', + streaming: true, + promptChars: 1200, + context: 'acme/demo', + responseChars: 5000, + }); + reporter.onParseComplete({ + stage: 'repository_analysis', + model: 'gpt-5.5', + provider: 'custom', + streaming: true, + promptChars: 1200, + context: 'acme/demo', + kind: 'repository_suggestion_list', + status: 'success', + parsed: { + version: '1', + kind: 'repository_suggestion_list', + status: 'success', + data: [ + { + id: 'docs-install', + title: 'Document local install', + summary: 'Clarify setup docs.', + targetFiles: [{ path: 'README.md', reason: 'Setup docs' }], + prPotentialScore: 82, + }, + ], + }, + }); + + const output = lines.join('\n'); + expect(output).toContain('LLM Interaction: repository_analysis'); + expect(output).toContain('Model: gpt-5.5'); + expect(output).toContain('Streaming: yes'); + expect(output).toContain('Prompt: 1,200 chars'); + expect(output).toContain('Context: acme/demo'); + expect(output).toContain('Assistant response received: 5,000 chars'); + expect(output).toContain('Repository suggestions: 1'); + expect(output).toContain('Document local install'); + expect(output).toContain('score 82'); + expect(output).toContain('README.md'); + expect(output).not.toContain('{"version"'); + }); + + test('raw mode buffers chunks and flushes original assistant output', () => { + const lines: string[] = []; + const reporter = new BufferedLLMInteractionReporter({ + write: (line) => lines.push(line), + chunkFlushChars: 10, + mode: 'raw', + }); + + reporter.onRequestStart({ + stage: 'patch_draft', + model: 'gpt-5.5', + provider: 'custom', + streaming: true, + promptChars: 1200, + context: 'acme/demo#42', + }); + reporter.onResponseChunk('{"hello":'); + reporter.onResponseChunk('"world"}'); + reporter.onResponseComplete({ + stage: 'patch_draft', + model: 'gpt-5.5', + provider: 'custom', + streaming: true, + promptChars: 1200, + context: 'acme/demo#42', + responseChars: 17, + }); + + const output = lines.join('\n'); + expect(output).toContain('Assistant:'); + expect(output).toContain('{"hello":"world"}'); + }); + + test('factory returns a no-op reporter when disabled', () => { + const lines: string[] = []; + const reporter = createLLMInteractionReporter(false, { write: (line) => lines.push(line) }); + + reporter.onRequestStart({ + stage: 'daily_report', + model: 'gpt-5.5', + provider: 'openai', + streaming: false, + promptChars: 10, + }); + + expect(lines).toEqual([]); + }); +}); diff --git a/test/llm.test.ts b/test/llm.test.ts index 9b2a217..ae4d72f 100644 --- a/test/llm.test.ts +++ b/test/llm.test.ts @@ -1,8 +1,12 @@ import { describe, expect, test } from 'bun:test'; import { LLMService } from '../src/services/llm.js'; import type { StructuredOutputStatus } from '../src/contracts/index.js'; -import type { ImplementationDraft, MatchedIssue } from '../src/types/index.js'; +import type { ImplementationDraft, LLMProvider, MatchedIssue } from '../src/types/index.js'; import { createIssue, createMemory, createRankedIssue, createWorkspace } from './helpers/factories.js'; +import type { + LLMInteractionEvent, + LLMInteractionReporter, +} from '../src/infra/llm-interaction-reporter.js'; interface LLMServiceInternals { validateConnection(): Promise; @@ -94,6 +98,53 @@ interface LLMServiceInternals { formatRepoMemory(memory: ReturnType): string; } +type RecordedInteractionEvent = + | { type: 'start'; event: LLMInteractionEvent } + | { type: 'chunk'; chunk: string } + | { type: 'complete'; event: LLMInteractionEvent & { responseChars: number } } + | { type: 'parse'; event: LLMInteractionEvent & { kind: string; status: string; parsed?: unknown } } + | { type: 'repair'; event: LLMInteractionEvent & { error: string } }; + +function createRecordingReporter(events: RecordedInteractionEvent[]): LLMInteractionReporter { + return { + onRequestStart: (event) => events.push({ type: 'start', event }), + onResponseChunk: (chunk) => events.push({ type: 'chunk', chunk }), + onResponseComplete: (event) => events.push({ type: 'complete', event }), + onParseComplete: (event) => events.push({ type: 'parse', event }), + onRepairStart: (event) => events.push({ type: 'repair', event }), + }; +} + +type InitializableService = LLMServiceInternals & { + initialize( + apiKey: string, + baseUrl: string, + modelName?: string, + apiHeaders?: Record, + provider?: LLMProvider, + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh', + stream?: boolean, + showInteraction?: boolean, + interactionReporter?: LLMInteractionReporter, + interactionMode?: 'summary' | 'raw', + ): void; + generateDailyReport(issueAnalysis: string): Promise; + generatePatchDraft( + issue: ReturnType, + workspace: ReturnType, + memory: ReturnType, + ): Promise<{ + status: StructuredOutputStatus; + data: { + goal: string; + targetFiles: Array<{ path: string; reason: string }>; + proposedChanges: Array<{ title: string; details: string; files: string[] }>; + risks: string[]; + validationNotes: string[]; + }; + }>; +}; + describe('LLMService repository suggestion parsing', () => { test('parses structured repository suggestions and keeps the highest scoring duplicate', () => { const service = new LLMService() as unknown as LLMServiceInternals; @@ -675,6 +726,325 @@ describe('LLMService reasoning effort requests', () => { }); }); +describe('LLMService interaction reporting', () => { + test('emits non-streaming request and response lifecycle events when enabled', async () => { + const events: RecordedInteractionEvent[] = []; + const service = new LLMService() as unknown as InitializableService; + const payloads: Array> = []; + + service.initialize( + 'sk-test', + 'https://api.openai.com/v1', + 'gpt-5.5', + {}, + 'openai', + 'none', + false, + true, + createRecordingReporter(events), + ); + service.client = { + chat: { + completions: { + create: async (payload) => { + payloads.push(payload); + return { choices: [{ message: { content: 'done' } }] }; + }, + }, + }, + }; + + const content = await service.generateDailyReport('issue analysis'); + + expect(content).toBe('done'); + expect(payloads[0]).not.toHaveProperty('stream'); + expect(events).toHaveLength(2); + expect(events[0]).toEqual({ + type: 'start', + event: expect.objectContaining({ + stage: 'daily_report', + model: 'gpt-5.5', + provider: 'openai', + streaming: false, + promptChars: expect.any(Number), + }), + }); + expect(events[1]).toEqual({ + type: 'complete', + event: expect.objectContaining({ + stage: 'daily_report', + responseChars: 4, + }), + }); + }); + + test('does not enable streaming transport when only interaction output is enabled', async () => { + const events: RecordedInteractionEvent[] = []; + const service = new LLMService() as unknown as InitializableService; + const payloads: Array> = []; + + service.initialize( + 'sk-test', + 'https://api.openai.com/v1', + 'gpt-5.5', + {}, + 'openai', + 'none', + false, + true, + createRecordingReporter(events), + ); + service.client = { + chat: { + completions: { + create: async (payload) => { + payloads.push(payload); + return { choices: [{ message: { content: 'done' } }] }; + }, + }, + }, + }; + + await service.generateDailyReport('issue analysis'); + + expect(payloads[0]).not.toHaveProperty('stream'); + expect(payloads[0]).not.toHaveProperty('stream_options'); + expect(events.find((event) => event.type === 'start')).toEqual({ + type: 'start', + event: expect.objectContaining({ + streaming: false, + }), + }); + }); + + test('emits streamed chunk events when raw interaction mode is enabled', async () => { + const events: RecordedInteractionEvent[] = []; + const service = new LLMService() as unknown as InitializableService; + + async function* streamChunks() { + yield { choices: [{ delta: { content: 'hel' } }] }; + yield { choices: [{ delta: { content: 'lo' } }] }; + yield { choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }; + } + + service.initialize( + 'sk-test', + 'https://api.openai.com/v1', + 'gpt-5.5', + {}, + 'openai', + 'none', + true, + true, + createRecordingReporter(events), + 'raw', + ); + service.client = { + chat: { + completions: { + create: () => streamChunks(), + }, + }, + }; + + const content = await service.generateDailyReport('issue analysis'); + + expect(content).toBe('hello'); + expect(events.filter((event) => event.type === 'chunk')).toEqual([ + { type: 'chunk', chunk: 'hel' }, + { type: 'chunk', chunk: 'lo' }, + ]); + expect(events.find((event) => event.type === 'start')).toEqual({ + type: 'start', + event: expect.objectContaining({ + stage: 'daily_report', + streaming: true, + }), + }); + expect(events.find((event) => event.type === 'complete')).toEqual({ + type: 'complete', + event: expect.objectContaining({ + responseChars: 5, + }), + }); + }); + + test('suppresses raw chunk events in summary interaction mode', async () => { + const events: RecordedInteractionEvent[] = []; + const service = new LLMService() as unknown as InitializableService; + + async function* streamChunks() { + yield { choices: [{ delta: { content: '{"version":"1",' } }] }; + yield { choices: [{ delta: { content: '"kind":"repository_suggestion_list"}' } }] }; + } + + service.initialize( + 'sk-test', + 'https://api.openai.com/v1', + 'gpt-5.5', + {}, + 'openai', + 'none', + true, + true, + createRecordingReporter(events), + 'summary', + ); + service.client = { + chat: { + completions: { + create: () => streamChunks(), + }, + }, + }; + + const content = await service.generateDailyReport('issue analysis'); + + expect(content).toBe('{"version":"1","kind":"repository_suggestion_list"}'); + expect(events.filter((event) => event.type === 'chunk')).toEqual([]); + expect(events.find((event) => event.type === 'complete')).toEqual({ + type: 'complete', + event: expect.objectContaining({ + responseChars: 51, + }), + }); + }); + + test('emits parse and repair lifecycle events for structured output', async () => { + const events: RecordedInteractionEvent[] = []; + const service = new LLMService() as unknown as InitializableService; + let requestCount = 0; + + service.initialize( + 'sk-test', + 'https://api.openai.com/v1', + 'gpt-5.5', + {}, + 'openai', + 'none', + false, + true, + createRecordingReporter(events), + ); + service.client = { + chat: { + completions: { + create: async () => { + requestCount += 1; + if (requestCount === 1) { + return { + choices: [{ message: { content: 'Plan: update the button component and tests.' } }], + }; + } + + return { + choices: [{ + message: { + content: JSON.stringify({ + version: '1', + kind: 'patch_draft', + status: 'success', + data: { + goal: 'Add accessible labels to icon-only buttons', + targetFiles: [ + { + path: 'src/components/IconButton.tsx', + reason: 'Primary component logic', + }, + ], + proposedChanges: [ + { + title: 'Update button API', + details: 'Require an accessible label for icon-only rendering.', + files: ['src/components/IconButton.tsx'], + }, + ], + risks: ['Consumer code may rely on current behavior'], + validationNotes: ['Run bun test after the patch'], + }, + }), + }, + }], + }; + }, + }, + }, + }; + + const draft = await service.generatePatchDraft( + createRankedIssue(), + createWorkspace({ validationCommands: createWorkspace().testCommands }), + createMemory(), + ); + + expect(draft.status).toBe('success'); + expect(events.find((event) => event.type === 'repair')).toEqual({ + type: 'repair', + event: expect.objectContaining({ + stage: 'patch_draft', + context: 'acme/demo#42', + error: expect.stringContaining('parseable JSON object'), + }), + }); + expect(events.filter((event) => event.type === 'parse')).toEqual([ + { + type: 'parse', + event: expect.objectContaining({ + stage: 'patch_draft', + kind: 'patch_draft', + status: 'success', + parsed: expect.objectContaining({ + kind: 'patch_draft', + status: 'success', + }), + }), + }, + ]); + }); + + test('ignores reporter failures so LLM calls can still complete', async () => { + const service = new LLMService() as unknown as InitializableService; + const throwingReporter: LLMInteractionReporter = { + onRequestStart: () => { + throw new Error('reporter failed'); + }, + onResponseChunk: () => { + throw new Error('reporter failed'); + }, + onResponseComplete: () => { + throw new Error('reporter failed'); + }, + onParseComplete: () => { + throw new Error('reporter failed'); + }, + onRepairStart: () => { + throw new Error('reporter failed'); + }, + }; + + service.initialize( + 'sk-test', + 'https://api.openai.com/v1', + 'gpt-5.5', + {}, + 'openai', + 'none', + false, + true, + throwingReporter, + ); + service.client = { + chat: { + completions: { + create: async () => ({ choices: [{ message: { content: 'done' } }] }), + }, + }, + }; + + await expect(service.generateDailyReport('issue analysis')).resolves.toBe('done'); + }); +}); + describe('LLMService issue scoring response parsing', () => { test('parses structured matched issues and sorts by score descending', () => { const service = new LLMService() as unknown as LLMServiceInternals; diff --git a/test/state-services.test.ts b/test/state-services.test.ts index 4a5979e..881d8f9 100644 --- a/test/state-services.test.ts +++ b/test/state-services.test.ts @@ -92,10 +92,29 @@ describe('stateful services', () => { expect(loaded.github.username).toBe('partial-user'); expect(loaded.llm.modelName).toBe('gpt-4o-mini'); + expect(loaded.llm.showInteraction).toBe(false); + expect(loaded.llm.interactionMode).toBe('summary'); expect(loaded.automation.enabled).toBe(false); expect(loaded.automation.scheduleTime).toBe('09:00'); }); + test('config service defaults LLM interaction output to false', async () => { + const service = new ConfigService(); + const configPath = service.getConfigPath(); + + rmSync(join(tempRoot, '.config'), { recursive: true, force: true }); + mkdirSync(join(tempRoot, '.config', 'openmeta'), { recursive: true }); + writeFileSync(configPath, JSON.stringify({ + github: { pat: '', username: '' }, + llm: { apiKey: '', apiBaseUrl: 'https://example.com/v1', modelName: 'test-model' }, + }), 'utf-8'); + + const loaded = await service.load(); + + expect(loaded.llm.showInteraction).toBe(false); + expect(loaded.llm.interactionMode).toBe('summary'); + }); + test('memory service persists repo memory snapshots', () => { const nextMemory = memoryService.update(createRankedIssue(), createWorkspace()); const loadedMemory = memoryService.load('acme/demo');