diff --git a/cmd/gh-aw/compile_flags_test.go b/cmd/gh-aw/compile_flags_test.go index c8488c601a7..c4f64f34a74 100644 --- a/cmd/gh-aw/compile_flags_test.go +++ b/cmd/gh-aw/compile_flags_test.go @@ -46,3 +46,20 @@ func TestCompileOptionsPropagateForceRefreshContainerPins(t *testing.T) { t.Fatal("expected ForceRefreshContainerPins to be propagated to CompileConfig") } } + +func TestCompileOptionsPropagateModels(t *testing.T) { + t.Parallel() + + modelsFlag := compileCmd.Flags().Lookup("models") + if modelsFlag == nil { + t.Fatal("expected --models flag on compile command") + } + if modelsFlag.DefValue != "false" { + t.Fatalf("expected --models default to be false, got %s", modelsFlag.DefValue) + } + + config := (&compileCmdOptions{models: true}).toCompileConfig(nil) + if !config.Models { + t.Fatal("expected Models to be propagated to CompileConfig") + } +} diff --git a/cmd/gh-aw/main.go b/cmd/gh-aw/main.go index ed20a246648..ced2c19665d 100644 --- a/cmd/gh-aw/main.go +++ b/cmd/gh-aw/main.go @@ -428,6 +428,7 @@ type compileCmdOptions struct { showAllErrors bool fix bool stats bool + models bool failFast bool noCheckUpdate bool staged bool @@ -472,6 +473,7 @@ func getCompileCmdOptions(cmd *cobra.Command) compileCmdOptions { showAllErrors, _ := cmd.Flags().GetBool("show-all") fix, _ := cmd.Flags().GetBool("fix") stats, _ := cmd.Flags().GetBool("stats") + models, _ := cmd.Flags().GetBool("models") failFast, _ := cmd.Flags().GetBool("fail-fast") noCheckUpdate, _ := cmd.Flags().GetBool("no-check-update") scheduleSeed, _ := cmd.Flags().GetString("schedule-seed") @@ -488,7 +490,7 @@ func getCompileCmdOptions(cmd *cobra.Command) compileCmdOptions { validate: validate, watch: watch, noEmit: noEmit, purge: purge, strict: strict, trial: trial, dependabot: dependabot, forceOverwrite: forceOverwrite, refreshStopTime: refreshStopTime, forceRefreshActionPins: forceRefreshActionPins, forceRefreshContainerPins: forceRefreshContainerPins, allowActionRefs: allowActionRefs, zizmor: zizmor, poutine: poutine, actionlint: actionlint, runnerGuard: runnerGuard, syft: syft, grype: grype, grant: grant, yamllint: yamllint, shellcheck: shellcheck, - jsonOutput: jsonOutput, showAllErrors: showAllErrors, fix: fix, stats: stats, failFast: failFast, noCheckUpdate: noCheckUpdate, + jsonOutput: jsonOutput, showAllErrors: showAllErrors, fix: fix, stats: stats, models: models, failFast: failFast, noCheckUpdate: noCheckUpdate, staged: staged, approve: approve, validateImages: validateImages, ghes: ghes, verbose: verbose, useSamples: useSamples, } } @@ -521,7 +523,7 @@ func (o *compileCmdOptions) toCompileConfig(args []string) cli.CompileConfig { Dependabot: o.dependabot, ForceOverwrite: o.forceOverwrite, RefreshStopTime: o.refreshStopTime, ForceRefreshActionPins: o.forceRefreshActionPins, ForceRefreshContainerPins: o.forceRefreshContainerPins, AllowActionRefs: o.allowActionRefs, Zizmor: o.zizmor, Poutine: o.poutine, Actionlint: o.actionlint, RunnerGuard: o.runnerGuard, Syft: o.syft, Grype: o.grype, Grant: o.grant, Yamllint: o.yamllint, Shellcheck: o.shellcheck, JSONOutput: o.jsonOutput, ShowAllErrors: o.showAllErrors, - Stats: o.stats, FailFast: o.failFast, ScheduleSeed: o.scheduleSeed, Staged: o.staged, Approve: o.approve, + Stats: o.stats, Models: o.models, FailFast: o.failFast, ScheduleSeed: o.scheduleSeed, Staged: o.staged, Approve: o.approve, ValidateImages: o.validateImages, PriorManifestFile: o.priorManifestFile, GHESCompat: o.ghes, UseSamples: o.useSamples, } } @@ -546,7 +548,9 @@ func runCompileCmd(cmd *cobra.Command, args []string) error { return err } } - if _, err := cli.CompileWorkflows(cmd.Context(), opts.toCompileConfig(args)); err != nil { + config := opts.toCompileConfig(args) + cli.PrepareCompileModelValidation(cmd.Context(), &config) + if _, err := cli.CompileWorkflows(cmd.Context(), config); err != nil { return err } return nil @@ -793,6 +797,7 @@ func configureCompileToolFlags() { compileCmd.Flags().BoolP("json", "j", false, "Output results in JSON format") compileCmd.Flags().Bool("show-all", false, "Display all compilation errors instead of only the highest-priority subset (default: top 5)") compileCmd.Flags().Bool("stats", false, "Display statistics table sorted by workflow file size (shows jobs, steps, scripts, and shells)") + compileCmd.Flags().Bool("models", false, "Warn when models configured in models or engine.models are absent from the active model inventory") compileCmd.Flags().Bool("fail-fast", false, "Stop at the first validation error instead of collecting all errors") compileCmd.Flags().Bool("no-check-update", false, "Skip checking for gh-aw updates") compileCmd.Flags().String("schedule-seed", "", "Override the repository slug (owner/repo) used as seed for fuzzy schedule scattering (e.g., \"github/gh-aw\"). Bypasses git remote detection entirely. Use this when your git remote is not named \"origin\" and you have multiple remotes configured") diff --git a/docs/src/content/docs/setup/cli.md b/docs/src/content/docs/setup/cli.md index 006f85a67df..fac7cfc0267 100644 --- a/docs/src/content/docs/setup/cli.md +++ b/docs/src/content/docs/setup/cli.md @@ -363,10 +363,12 @@ If the repository root contains an [`aw.yml` manifest](/gh-aw/reference/aw-yml-p Unlike `gh aw upgrade`, `gh aw compile` does not run codemods unless you pass `--fix`. -**Options:** `--action-mode`, `--action-tag`, `--actionlint`, `--actions-repo`, `--allow-action-refs`, `--approve`, `--dependabot`, `--dir/-d`, `--engine/-e`, `--fail-fast`, `--fix`, `--force/-f`, `--force-refresh-action-pins`, `--force-refresh-container-pins`, `--gh-aw-ref`, `--ghes`, `--grant`, `--grype`, `--json/-j`, `--logical-repo/-l`, `--no-check-update`, `--no-emit`, `--poutine`, `--purge`, `--refresh-stop-time`, `--runner-guard`, `--schedule-seed`, `--shellcheck`, `--show-all`, `--staged`, `--stats`, `--strict`, `--syft`, `--trial`, `--validate`, `--validate-images`, `--watch/-w`, `--yamllint`, `--zizmor` +**Options:** `--action-mode`, `--action-tag`, `--actionlint`, `--actions-repo`, `--allow-action-refs`, `--approve`, `--dependabot`, `--dir/-d`, `--engine/-e`, `--fail-fast`, `--fix`, `--force/-f`, `--force-refresh-action-pins`, `--force-refresh-container-pins`, `--gh-aw-ref`, `--ghes`, `--grant`, `--grype`, `--json/-j`, `--logical-repo/-l`, `--models`, `--no-check-update`, `--no-emit`, `--poutine`, `--purge`, `--refresh-stop-time`, `--runner-guard`, `--schedule-seed`, `--shellcheck`, `--show-all`, `--staged`, `--stats`, `--strict`, `--syft`, `--trial`, `--validate`, `--validate-images`, `--watch/-w`, `--yamllint`, `--zizmor` **`--gh-aw-ref` flag:** Convenience alias for `--action-mode release --action-tag `. Accepts a branch name, tag, or commit SHA targeting the `github/gh-aw` repository. Branch and tag names are resolved to their full commit SHA at compile time, so the baked-in reference is immutable and reproducible. Useful for E2E-testing workflows compiled against a specific gh-aw revision. +**`--models` flag:** Refreshes the observed model inventory using the same data sources as `gh aw models`, then warns when `models.allowed`, `models.blocked`, or `engine.models` references an unknown model. Built-in and workflow model aliases are accepted. If no observed model data is available, the check is skipped. + **`--approve` flag:** When compiling a workflow that already has a lock file, the compiler enforces *safe update mode* — any newly added secrets or custom actions not present in the previous manifest require explicit approval. Pass `--approve` to accept these changes and regenerate the manifest baseline. On first compile (no existing lock file), enforcement is skipped automatically and `--approve` is not needed. **Error Reporting:** Displays detailed error messages with file paths, line numbers, column positions, and contextual code snippets. diff --git a/pkg/cli/compile_compiler_setup.go b/pkg/cli/compile_compiler_setup.go index 16764fbb8f8..ffc02061ecd 100644 --- a/pkg/cli/compile_compiler_setup.go +++ b/pkg/cli/compile_compiler_setup.go @@ -99,6 +99,11 @@ func createAndConfigureCompiler(config CompileConfig) *workflow.Compiler { workflow.WithEngineOverride(config.EngineOverride), workflow.WithFailFast(config.FailFast), ) + if config.activeModels != nil { + compiler.SetConfiguredModelValidator(func(data *workflow.WorkflowData) []string { + return unknownConfiguredModelMessages(data, config.activeModels) + }) + } compileCompilerSetupLog.Print("Created compiler instance") // Configure compiler flags @@ -153,6 +158,10 @@ func configureCompilerFlags(compiler *workflow.Compiler, config CompileConfig) { compiler.SetUseSamples(true) } + configureCompilerMaintenanceFlags(compiler, config) +} + +func configureCompilerMaintenanceFlags(compiler *workflow.Compiler, config CompileConfig) { // Set refresh stop time flag compiler.SetRefreshStopTime(config.RefreshStopTime) if config.RefreshStopTime { diff --git a/pkg/cli/compile_config.go b/pkg/cli/compile_config.go index 40f62904b80..038c145ed0d 100644 --- a/pkg/cli/compile_config.go +++ b/pkg/cli/compile_config.go @@ -36,12 +36,14 @@ type CompileConfig struct { ActionTag string // Pin action refs to this SHA or version tag (e.g. v1, ). Sets release mode unless ActionMode is already "action". Mutually exclusive with GHAwRef at the CLI layer. ActionsRepo string // Override the external actions repository (default: github/gh-aw-actions) Stats bool // Display statistics table sorted by file size + Models bool // Warn about configured models absent from the observed active model inventory FailFast bool // Stop at first error instead of collecting all errors ScheduleSeed string // Override repository slug used for fuzzy schedule scattering (e.g. owner/repo) Approve bool // Approve all safe update changes, skipping safe update enforcement regardless of strict mode setting. ValidateImages bool // Require Docker to be available for container image validation (fail instead of skipping when Docker is unavailable) PriorManifestFile string // Path to a JSON file containing pre-cached manifests (map[lockFile]*GHAWManifest) collected at MCP server startup; takes precedence over git HEAD / filesystem reads for safe update enforcement GHESCompat bool // Enable GHES-compatible v3 artifact actions (overrides aw.json ghes field) + activeModels *activeModelInventory } func (c CompileConfig) shellcheckEnabled() bool { diff --git a/pkg/cli/compile_model_validation.go b/pkg/cli/compile_model_validation.go new file mode 100644 index 00000000000..b8f3b68d3c9 --- /dev/null +++ b/pkg/cli/compile_model_validation.go @@ -0,0 +1,156 @@ +package cli + +import ( + "context" + "path" + "slices" + "strings" + + "github.com/github/gh-aw/pkg/modelsdev" + "github.com/github/gh-aw/pkg/workflow" +) + +type activeModelInventory struct { + models []string + aliases map[string]struct{} +} + +func buildActiveModelInventory(report modelsReport) *activeModelInventory { + if len(report.Observed) == 0 { + return nil + } + + models := make(map[string]struct{}) + for _, row := range report.Observed { + model := modelsdev.NormalizeComparableModelID(row.Model) + if model == "" { + continue + } + models[model] = struct{}{} + if row.Provider != "" { + provider := modelsdev.NormalizeProvider(row.Provider) + models[modelsdev.NormalizeComparableModelID(path.Join(provider, row.Model))] = struct{}{} + if provider == "github-copilot" { + for _, alias := range []string{"copilot", "github", "github_models"} { + models[modelsdev.NormalizeComparableModelID(path.Join(alias, row.Model))] = struct{}{} + } + } + } + } + + aliases := make(map[string]struct{}, len(report.Aliases)) + for _, row := range report.Aliases { + aliases[modelsdev.NormalizeComparableModelID(row.Alias)] = struct{}{} + } + + activeModels := make([]string, 0, len(models)) + for model := range models { + activeModels = append(activeModels, model) + } + slices.Sort(activeModels) + return &activeModelInventory{models: activeModels, aliases: aliases} +} + +func (i *activeModelInventory) contains(candidate string, workflowAliases map[string][]string) bool { + base, _, _ := strings.Cut(strings.TrimSpace(candidate), "?") + if base == "" || strings.Contains(base, "${{") { + return true + } + + normalized := modelsdev.NormalizeComparableModelID(base) + if _, ok := i.aliases[normalized]; ok { + return true + } + for alias := range workflowAliases { + if modelsdev.NormalizeComparableModelID(alias) == normalized { + return true + } + } + + for _, model := range i.models { + if matched, err := path.Match(normalized, model); err == nil && matched { + return true + } + } + return false +} + +func findUnknownConfiguredModels(data *workflow.WorkflowData, inventory *activeModelInventory) []ValidationIssue { + if data == nil || inventory == nil { + return nil + } + + candidates := make(map[string][]string) + add := func(field string, values []string) { + for _, value := range values { + if !inventory.contains(value, data.ModelMappings) { + candidates[value] = append(candidates[value], field) + } + } + } + + add("models.allowed", data.ModelPolicyAllowed) + add("models.blocked", data.ModelPolicyBlocked) + + if engine, ok := data.RawFrontmatter["engine"].(map[string]any); ok { + if models, ok := engine["models"].(map[string]any); ok { + if value, ok := models["default"].(string); ok { + add("engine.models.default", []string{value}) + } + add("engine.models.supported", stringSlice(models["supported"])) + } + } + + values := make([]string, 0, len(candidates)) + for value := range candidates { + values = append(values, value) + } + slices.Sort(values) + + warnings := make([]ValidationIssue, 0, len(values)) + for _, value := range values { + fields := candidates[value] + slices.Sort(fields) + warnings = append(warnings, ValidationIssue{ + Type: "unknown_model", + Message: "Model " + value + " referenced by " + strings.Join(fields, ", ") + " was not found in the active model inventory", + }) + } + return warnings +} + +// PrepareCompileModelValidation builds the active model inventory used by compile --models. +func PrepareCompileModelValidation(ctx context.Context, config *CompileConfig) { + if !config.Models { + return + } + report := buildModelsReport(ctx, modelsReportOptions{ + logsDir: defaultLogsOutputDir, + refreshObserved: true, + refreshCount: defaultModelsRefreshCount, + }) + config.activeModels = buildActiveModelInventory(report) +} + +func unknownConfiguredModelMessages(data *workflow.WorkflowData, inventory *activeModelInventory) []string { + issues := findUnknownConfiguredModels(data, inventory) + messages := make([]string, 0, len(issues)) + for _, issue := range issues { + messages = append(messages, issue.Message) + } + return messages +} + +func stringSlice(value any) []string { + raw, ok := value.([]any) + if !ok { + return nil + } + result := make([]string, 0, len(raw)) + for _, entry := range raw { + if text, ok := entry.(string); ok { + result = append(result, text) + } + } + return result +} diff --git a/pkg/cli/compile_model_validation_test.go b/pkg/cli/compile_model_validation_test.go new file mode 100644 index 00000000000..4f18e0a5778 --- /dev/null +++ b/pkg/cli/compile_model_validation_test.go @@ -0,0 +1,70 @@ +//go:build !integration + +package cli + +import ( + "testing" + + "github.com/github/gh-aw/pkg/workflow" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildActiveModelInventoryDoesNothingWithoutObservedData(t *testing.T) { + t.Parallel() + + assert.Nil(t, buildActiveModelInventory(modelsReport{ + Catalog: []modelCatalogRow{{Provider: "openai", Model: "gpt-5.4"}}, + Aliases: []modelAliasRow{{Alias: "large"}}, + })) +} + +func TestFindUnknownConfiguredModels(t *testing.T) { + t.Parallel() + + inventory := buildActiveModelInventory(modelsReport{ + Aliases: []modelAliasRow{{Alias: "large"}}, + Observed: []observedModelRow{ + {Provider: "github-copilot", Model: "gpt-5.4"}, + {Provider: "anthropic", Model: "claude-sonnet-4.6"}, + }, + }) + require.NotNil(t, inventory) + + data := &workflow.WorkflowData{ + ModelMappings: map[string][]string{"custom-alias": {"gpt-5.4"}}, + ModelPolicyAllowed: []string{ + "gpt-5.4", + "copilot/gpt-5.*", + "large", + "custom-alias", + "missing-policy-model", + }, + ModelPolicyBlocked: []string{"anthropic/claude-sonnet-4.6"}, + RawFrontmatter: map[string]any{ + "engine": map[string]any{ + "models": map[string]any{ + "default": "large", + "supported": []any{"gpt-5.4?effort=high", "${{ inputs.model }}", "missing-engine-model"}, + }, + }, + }, + } + + warnings := findUnknownConfiguredModels(data, inventory) + require.Len(t, warnings, 2) + assert.Equal(t, "unknown_model", warnings[0].Type) + assert.Contains(t, warnings[0].Message, "missing-engine-model") + assert.Contains(t, warnings[0].Message, "engine.models.supported") + assert.Contains(t, warnings[1].Message, "missing-policy-model") + assert.Contains(t, warnings[1].Message, "models.allowed") +} + +func TestFindUnknownConfiguredModelsSkipsMissingInventory(t *testing.T) { + t.Parallel() + + data := &workflow.WorkflowData{ + ModelPolicyAllowed: []string{"unknown"}, + } + assert.Empty(t, findUnknownConfiguredModels(data, nil)) +} diff --git a/pkg/cli/models_command.go b/pkg/cli/models_command.go index 6d33a0c6c69..40705f60b4c 100644 --- a/pkg/cli/models_command.go +++ b/pkg/cli/models_command.go @@ -19,6 +19,8 @@ import ( "github.com/spf13/cobra" ) +const defaultModelsRefreshCount = 20 + // NewModelsCommand creates the models command. func NewModelsCommand() *cobra.Command { cmd := &cobra.Command{ @@ -46,7 +48,7 @@ so recent awf-reflect data can be discovered before reporting.`, addJSONFlag(cmd) cmd.Flags().String("logs-dir", defaultLogsOutputDir, "Directory containing downloaded logs/artifacts") cmd.Flags().Bool("refresh-observed", true, "Attempt to refresh local observed-model artifacts before reporting") - cmd.Flags().Int("refresh-count", 20, "Maximum number of recent runs to inspect when refreshing observed models") + cmd.Flags().Int("refresh-count", defaultModelsRefreshCount, "Maximum number of recent runs to inspect when refreshing observed models") addRepoFlag(cmd) return cmd } @@ -82,6 +84,13 @@ type modelsReport struct { Warnings []string `json:"warnings,omitempty"` } +type modelsReportOptions struct { + logsDir string + refreshObserved bool + refreshCount int + repoOverride string +} + const maxAliasHints = 6 func runModelsCommand(cmd *cobra.Command) error { @@ -91,24 +100,12 @@ func runModelsCommand(cmd *cobra.Command) error { refreshCount, _ := cmd.Flags().GetInt("refresh-count") repoOverride, _ := cmd.Flags().GetString("repo") - warnings := make([]string, 0) - if refreshObserved { - if err := refreshObservedArtifacts(cmd.Context(), logsDir, refreshCount, repoOverride); err != nil { - warnings = append(warnings, "observed-model refresh failed: "+err.Error()) - } - } - - catalogRows := buildModelCatalogRows() - aliasRows, aliasMap := buildModelAliasRows() - observedRows, observedWarnings := collectObservedModelRows(logsDir, aliasMap) - warnings = append(warnings, observedWarnings...) - - report := modelsReport{ - Catalog: catalogRows, - Aliases: aliasRows, - Observed: observedRows, - Warnings: warnings, - } + report := buildModelsReport(cmd.Context(), modelsReportOptions{ + logsDir: logsDir, + refreshObserved: refreshObserved, + refreshCount: refreshCount, + repoOverride: repoOverride, + }) if jsonOutput { jsonBytes, err := marshalIndentJSONOrWrap(report, "models report") @@ -120,26 +117,47 @@ func runModelsCommand(cmd *cobra.Command) error { } fmt.Fprintln(os.Stdout, "Catalog Models") - fmt.Fprint(os.Stdout, console.RenderStruct(catalogRows)) + fmt.Fprint(os.Stdout, console.RenderStruct(report.Catalog)) fmt.Fprintln(os.Stdout) fmt.Fprintln(os.Stdout, "Model Aliases") - fmt.Fprint(os.Stdout, console.RenderStruct(aliasRows)) + fmt.Fprint(os.Stdout, console.RenderStruct(report.Aliases)) fmt.Fprintln(os.Stdout) fmt.Fprintln(os.Stdout, "Observed Models") - if len(observedRows) == 0 { + if len(report.Observed) == 0 { fmt.Fprintln(os.Stdout, "No observed models found in local logs/artifacts.") } else { - fmt.Fprint(os.Stdout, console.RenderStruct(observedRows)) + fmt.Fprint(os.Stdout, console.RenderStruct(report.Observed)) } - for _, warning := range warnings { + for _, warning := range report.Warnings { fmt.Fprintln(os.Stderr, warning) } return nil } +func buildModelsReport(ctx context.Context, opts modelsReportOptions) modelsReport { + warnings := make([]string, 0) + if opts.refreshObserved { + if err := refreshObservedArtifacts(ctx, opts.logsDir, opts.refreshCount, opts.repoOverride); err != nil { + warnings = append(warnings, "observed-model refresh failed: "+err.Error()) + } + } + + catalogRows := buildModelCatalogRows() + aliasRows, aliasMap := buildModelAliasRows() + observedRows, observedWarnings := collectObservedModelRows(opts.logsDir, aliasMap) + warnings = append(warnings, observedWarnings...) + + return modelsReport{ + Catalog: catalogRows, + Aliases: aliasRows, + Observed: observedRows, + Warnings: warnings, + } +} + func refreshObservedArtifacts(ctx context.Context, logsDir string, refreshCount int, repoOverride string) error { if refreshCount <= 0 { - refreshCount = 20 + refreshCount = defaultModelsRefreshCount } return DownloadWorkflowLogs(ctx, LogsDownloadOptions{ Count: refreshCount, diff --git a/pkg/workflow/compiler_mutators.go b/pkg/workflow/compiler_mutators.go index 1a5abc31f0a..1b447e7a5b9 100644 --- a/pkg/workflow/compiler_mutators.go +++ b/pkg/workflow/compiler_mutators.go @@ -179,6 +179,11 @@ func (c *Compiler) IncrementWarningCount() { c.warningCount++ } +// SetConfiguredModelValidator configures optional validation against an external active model inventory. +func (c *Compiler) SetConfiguredModelValidator(validator func(data *WorkflowData) []string) { + c.configuredModelValidator = validator +} + // GetWarningCount returns the current warning count func (c *Compiler) GetWarningCount() int { return c.warningCount diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 550290a80da..85ff90cacd1 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -130,6 +130,7 @@ func (c *Compiler) validateWorkflowBuildContext(ctx *workflowBuildContext) error if err := c.validateWorkflowModelAliasMap(ctx); err != nil { return err } + c.warnUnknownConfiguredModels(ctx.workflowData, ctx.cleanPath) if err := c.validateWorkflowEngineSettings(ctx.cleanPath, ctx.workflowData); err != nil { return err } diff --git a/pkg/workflow/compiler_types.go b/pkg/workflow/compiler_types.go index c25788ec36e..2563bf35a8b 100644 --- a/pkg/workflow/compiler_types.go +++ b/pkg/workflow/compiler_types.go @@ -80,7 +80,8 @@ type Compiler struct { // buildInitialWorkflowData for the workflow's configured model; any returned pricing is merged // into WorkflowData.ModelCosts so it is embedded in GH_AW_INFO_MODEL_COSTS in the lock.yml. // Injected by the cli package (which has access to the embedded catalog and models.dev download). - modelPricingResolver func(ctx context.Context, provider, model string) (map[string]float64, bool) + modelPricingResolver func(ctx context.Context, provider, model string) (map[string]float64, bool) + configuredModelValidator func(data *WorkflowData) []string } type allowedDomain struct { diff --git a/pkg/workflow/configured_model_validation.go b/pkg/workflow/configured_model_validation.go new file mode 100644 index 00000000000..9ba76371304 --- /dev/null +++ b/pkg/workflow/configured_model_validation.go @@ -0,0 +1,19 @@ +package workflow + +import ( + "fmt" + "os" + + "github.com/github/gh-aw/pkg/console" +) + +func (c *Compiler) warnUnknownConfiguredModels(data *WorkflowData, markdownPath string) { + if c.configuredModelValidator == nil { + return + } + for _, warning := range c.configuredModelValidator(data) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage( + formatCompilerMessage(markdownPath, "warning", warning))) + c.IncrementWarningCount() + } +} diff --git a/pkg/workflow/configured_model_validation_test.go b/pkg/workflow/configured_model_validation_test.go new file mode 100644 index 00000000000..ec2097ed9de --- /dev/null +++ b/pkg/workflow/configured_model_validation_test.go @@ -0,0 +1,46 @@ +//go:build !integration + +package workflow + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWarnUnknownConfiguredModels(t *testing.T) { + // This test redirects the process-wide stderr stream and cannot run in parallel. + compiler := NewCompiler() + compiler.SetConfiguredModelValidator(func(data *WorkflowData) []string { + assert.Equal(t, "test", data.WorkflowID) + return []string{"Model missing was not found in the active model inventory"} + }) + + oldStderr := os.Stderr + reader, writer, err := os.Pipe() + require.NoError(t, err) + os.Stderr = writer + defer func() { + os.Stderr = oldStderr + }() + + compiler.warnUnknownConfiguredModels(&WorkflowData{WorkflowID: "test"}, "test.md") + require.NoError(t, writer.Close()) + output, err := io.ReadAll(reader) + require.NoError(t, err) + + assert.Equal(t, 1, compiler.GetWarningCount()) + assert.Contains(t, string(bytes.TrimSpace(output)), "test.md: warning: Model missing") +} + +func TestWarnUnknownConfiguredModelsWithoutInventory(t *testing.T) { + t.Parallel() + + compiler := NewCompiler() + compiler.warnUnknownConfiguredModels(&WorkflowData{}, "test.md") + assert.Zero(t, compiler.GetWarningCount()) +}