diff --git a/skills/changelogs/fix-changelog/SKILL.md b/skills/changelogs/fix-changelog/SKILL.md index 5235499..f93597f 100644 --- a/skills/changelogs/fix-changelog/SKILL.md +++ b/skills/changelogs/fix-changelog/SKILL.md @@ -10,8 +10,7 @@ sources: - https://www.elastic.co/docs/contribute-docs/content-types/changelogs - https://elastic.github.io/docs-builder/syntax/links/ - https://elastic.github.io/docs-builder/syntax/code/ - - +--- You are a changelog writing assistant for Elastic documentation. You suggest improved text for changelog fields and help draft content for new changelogs. You do not create files — file creation is always done via `docs-builder changelog add`. diff --git a/skills/changelogs/fix-changelog/evals/evals.json b/skills/changelogs/fix-changelog/evals/evals.json index 6441360..cd6b23e 100644 --- a/skills/changelogs/fix-changelog/evals/evals.json +++ b/skills/changelogs/fix-changelog/evals/evals.json @@ -3,17 +3,28 @@ "evals": [ { "id": 1, + "prompt": "Review this changelog file for improvements: ./perfect-changelog.yaml. It's a feature addition for Kibana with clear title and description.", + "expected_output": "No significant improvements needed; changelog already follows best practices with clear user-focused title, appropriate type alignment, proper YAML formatting, and comprehensive description.", + "expectations": [ + "Recognizes the changelog is already well-written", + "Either reports no issues found or only minor stylistic suggestions", + "Does not suggest unnecessary changes to good content", + "Maintains proper confidence scoring for high-quality input" + ] + }, + { + "id": 2, "prompt": "Fix this changelog title that has UI formatting issues: 'Update Service Inventory page for better Machine Learning display'", "expected_output": "Suggestion showing proper UI element formatting with \"Service Inventory\" quoted (UI label) but Machine Learning capitalized (feature name), not quoted", "expectations": [ "Suggests quoting 'Service Inventory' as UI element: \"Service Inventory\"", "Suggests capitalizing 'Machine Learning' as feature name (not quoting)", - "Includes article: 'the \"Service Inventory\" page'", + "Includes article: 'the \"Service Inventory\" page'", "Explains UI vs feature name formatting distinction" ] }, { - "id": 2, + "id": 3, "prompt": "Fix this changelog where accuracy conflicts with style: Title 'Improve dashboard performance' but PR shows it actually fixes a memory leak bug that was causing crashes", "expected_output": "Suggestion prioritizes accuracy over style, recommending title change to reflect bug fix nature and possibly changing type to bug-fix", "expectations": [ @@ -24,7 +35,7 @@ ] }, { - "id": 3, + "id": 4, "prompt": "Fix changelog with uncertain formatting: 'Enable Advanced Settings in Kibana for index.refresh_interval configuration'", "expected_output": "Confidence tracking documents uncertainty about 'Advanced Settings' (UI vs feature) and properly formats index.refresh_interval with backticks", "expectations": [ @@ -35,7 +46,117 @@ ] }, { - "id": 4, + "id": 5, + "prompt": "Fix this changelog with multiple schema errors: missing type field, invalid product ID 'invalid-product', and unquoted title containing colons: Fix API endpoint: authentication now required", + "expected_output": "Comprehensive fix suggestions addressing schema violations: adds required type field, corrects product ID to valid value, properly quotes title to prevent YAML parsing errors, plus a docs-builder changelog add command.", + "expectations": [ + "Identifies and fixes all schema violations", + "Suggests valid product ID from canonical list", + "Properly quotes title containing colons to prevent parse errors", + "Provides ready-to-copy docs-builder command with proper escaping", + "High confidence for schema fixes, medium/low confidence for field content suggestions" + ] + }, + { + "id": 6, + "prompt": "This breaking-change changelog is missing required impact and action fields. Type: breaking-change, Title: Remove deprecated API endpoints, Description: Several legacy endpoints are no longer available.", + "expected_output": "Identifies missing required fields for breaking-change type and suggests comprehensive impact and action content that explains who is affected and provides migration steps.", + "expectations": [ + "Flags missing impact and action fields as required for breaking-change type", + "Suggests meaningful impact describing affected users and use cases", + "Provides actionable migration steps in the action field", + "Recommends adding subtype field for better categorization", + "Medium confidence noting limited context for migration details" + ] + }, + { + "id": 7, + "prompt": "Fix type-title alignment: Type is 'bug-fix' but title says 'Improve query performance in Elasticsearch'. The change optimized existing functionality that was working but slow.", + "expected_output": "Type-title alignment mismatch detected with two options: Option A keeps bug-fix type and rewrites title to focus on what was broken, Option B changes type to enhancement and keeps performance-focused title.", + "expectations": [ + "Identifies type-title alignment mismatch clearly", + "Provides Option A: bug-fix type with title like 'Fix slow query performance'", + "Provides Option B: enhancement type with title like 'Improve query performance'", + "Includes recommendation explaining which option fits better based on context", + "Medium confidence due to ambiguous nature of performance improvements" + ] + }, + { + "id": 8, + "prompt": "Rewrite this overly technical title for user impact: 'Fix splitValue nullability coercion when constructing ColorSeries for ES|QL visualization rendering'", + "expected_output": "Technical content assessment identifies implementation-focused title and suggests user-focused rewrite emphasizing visible symptoms like 'Fix chart display issues in ES|QL query visualizations' with explanation of the improvement.", + "expectations": [ + "Flags title as too technical and implementation-focused", + "Suggests user-focused alternative describing visible symptoms", + "Maintains ES|QL context but explains user impact", + "Explains why the rewrite better serves users", + "High confidence for technical content rewrite recommendation" + ] + }, + { + "id": 9, + "prompt": "Fix quality issues in this changelog: Title: 'feat: Bug fixes', Description: 'See PR for details', Type: feature, with development prefix and vague content.", + "expected_output": "Multiple content quality fixes: strips 'feat:' development prefix, makes title specific and actionable, replaces generic description with meaningful user impact, suggests aligning type with actual content.", + "expectations": [ + "Strips development label 'feat:' from title", + "Identifies 'Bug fixes' as too vague for feature type", + "Flags 'See PR for details' as low-value description", + "Suggests specific title and meaningful description", + "Notes type-content mismatch (bug fixes vs feature type)", + "Low confidence due to lack of PR context for specific improvements" + ] + }, + { + "id": 10, + "prompt": "Fix YAML formatting in changelog with unquoted title containing special characters: Fix auth header: Bearer token format, and description with code identifiers written as plain text: Updates the validateToken method behavior.", + "expected_output": "YAML formatting fixes including proper quoting for special characters and backticks around code identifiers, plus improved technical terminology for user clarity.", + "expectations": [ + "Quotes title properly due to colon followed by space", + "Adds backticks around 'validateToken' method name", + "Explains YAML quoting requirements to prevent parse errors", + "Suggests more user-focused description of method behavior impact", + "High confidence for formatting fixes, medium for content improvements" + ] + }, + { + "id": 11, + "prompt": "Review changelog with invalid area 'Dashboards' (note: this area doesn't exist in the current repository's docs/changelog.yml areas list).", + "expected_output": "Repository-aware validation flags invalid area name and suggests correct area from the repository's canonical list, noting the difference between repository-specific and generic validation.", + "expectations": [ + "Identifies area as invalid based on repository configuration", + "Suggests valid area alternatives from docs/changelog.yml", + "Notes successful repository config loading in confidence section", + "High confidence for area validation when repository config available" + ] + }, + { + "id": 12, + "prompt": "Process this directory of changelog files: ./mixed-changelogs/ containing mixed quality files from different products (Kibana cloud-serverless, Elasticsearch, docs-builder).", + "expected_output": "Directory processing mode with individual file assessments, summary of files needing improvements, and recognition of different product types and area taxonomies across the mixed repository examples.", + "expectations": [ + "Processes multiple files in directory mode", + "Provides individual assessments for each file", + "Summarizes count of files processed and files needing improvements", + "Handles different product types (cloud-serverless, elasticsearch, docs-builder)", + "Recognizes different area taxonomies (Kibana UX vs ES|QL vs API/CLI)", + "Does not apply changes without user confirmation" + ] + }, + { + "id": 13, + "prompt": "Create new changelog for: 'Added support for custom authentication providers in Elasticsearch Serverless, allowing users to configure SAML and OIDC integration with enhanced security options including role mapping.'", + "expected_output": "Mode C operation providing suggested field content and ready-to-copy docs-builder changelog add command with proper escaping for shell execution.", + "expectations": [ + "Suggests appropriate type (likely 'feature' for new capability)", + "Creates clear, user-focused title under 80 characters", + "Provides comprehensive description under 600 characters", + "Outputs properly formatted docs-builder changelog add command", + "Escapes backticks and quotes correctly for shell execution", + "Notes that products, PRs, issues must be provided separately" + ] + }, + { + "id": 14, "prompt": "Fix changelog title with ambiguous acronym: 'fix: KI details flyout pushes page content in dark mode'. No PR context provided.", "expected_output": "Flags 'KI' acronym expansion as uncertain, asks for clarification, and documents acronym uncertainty in confidence section rather than guessing the expansion", "expectations": [ @@ -48,7 +169,7 @@ ] }, { - "id": 5, + "id": 15, "prompt": "Fix changelog with feature prefix: 'File upload: Fixes disabling data view creation based on user capabilities'", "expected_output": "Detects feature/app prefix pattern and suggests contextual integration alternatives using integration templates", "expectations": [ @@ -60,7 +181,7 @@ ] }, { - "id": 6, + "id": 16, "prompt": "Fix changelog with technical colon usage: 'Fix bug with Auth header needing \"Authorization: Bearer \"'", "expected_output": "Correctly skips technical term and does NOT flag as feature prefix, focusing on other improvements", "expectations": [ @@ -71,7 +192,7 @@ ] }, { - "id": 7, + "id": 17, "prompt": "Fix changelog with ambiguous prefix: 'Security: Enable new authentication method'", "expected_output": "Detects potential feature prefix but shows medium confidence due to ambiguous context", "expectations": [ @@ -83,4 +204,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/skills/changelogs/review-changelog/evals/evals.json b/skills/changelogs/review-changelog/evals/evals.json index 9293357..af3e424 100644 --- a/skills/changelogs/review-changelog/evals/evals.json +++ b/skills/changelogs/review-changelog/evals/evals.json @@ -1,8 +1,20 @@ { - "skill_name": "docs-review-changelog", + "skill_name": "docs-review-changelog", "evals": [ { "id": 1, + "prompt": "Review this perfect changelog file: ./perfect-changelog.yaml with proper title, valid type, correct product, appropriate areas, and well-written description.", + "expected_output": "Clean review report stating no issues found, proper markdown structure with Summary section showing 0 schema errors, 0 quality warnings, 0 formatting warnings.", + "expectations": [ + "Reports no schema errors for valid required fields", + "Reports no quality warnings for well-written content", + "Reports no formatting warnings for proper YAML structure", + "Uses proper markdown report format with Summary section", + "Explicitly states the file has no issues rather than omitting sections" + ] + }, + { + "id": 2, "prompt": "Review this changelog with UI formatting issues: title: 'Update Service Inventory to improve **Machine Learning** display' description: 'Changes to Advanced Settings panel'", "expected_output": "Flags UI formatting issues: Service Inventory should be quoted, Machine Learning shouldn't be bolded or quoted (feature name), Advanced Settings panel formatting unclear", "expectations": [ @@ -13,29 +25,127 @@ ] }, { - "id": 2, - "prompt": "Review changelog with Elastic context issues: title: 'Add support for ESQL queries in Observability' description: 'Enables ESQL functionality for monitoring use cases'", + "id": 3, + "prompt": "Review changelog with Elastic context issues: title: 'Add support for ESQL queries in Observability' description: 'Enables ESQL functionality for monitoring use cases'", "expected_output": "Applies Elastic software context to validate ES|QL terminology and Observability product reference", "expectations": [ "Notes inconsistent 'ESQL' format (should be 'ES|QL')", - "Validates 'Observability' as legitimate Elastic product context", + "Validates 'Observability' as legitimate Elastic product context", "Shows software context awareness in terminology checking", "Does not flag legitimate Elastic product references as marketing" ] }, { - "id": 3, + "id": 4, "prompt": "Review changelog when canonical guidance fails to load: title: 'Fix dashboard loading' with repository config not found", - "expected_output": "Includes confidence assessment showing failed resource loading and its impact on review accuracy", + "expected_output": "Includes confidence assessment showing failed resource loading and its impact on review accuracy", "expectations": [ "Includes 'Confidence Assessment' section in output", - "Shows '✗ Failed' status for canonical guidance or repository config", + "Shows '✗ Failed' status for canonical guidance or repository config", "Notes 'Medium' or 'Low' validation confidence due to missing resources", "Explains review limitations due to resource loading failures" ] }, { - "id": 4, + "id": 5, + "prompt": "Review changelog with multiple schema errors: missing type field, unknown product 'invalid-product-id', invalid type 'bugfix', and unquoted title with colons causing YAML parse issues.", + "expected_output": "Schema errors section listing each violation: missing required type field, unknown product ID, invalid type value, and YAML quoting requirements for special characters.", + "expectations": [ + "Identifies missing type as schema error", + "Flags unknown product ID against canonical products.yml", + "Catches invalid type 'bugfix' (should be 'bug-fix')", + "Reports YAML quoting issue for unquoted colons", + "Uses proper markdown structure with Schema errors section", + "Summary shows multiple schema errors count" + ] + }, + { + "id": 6, + "prompt": "Review breaking-change changelog missing required impact and action fields: Type: breaking-change, Title: Remove legacy API, Description: Removes old endpoints.", + "expected_output": "Schema errors section flagging missing required impact and action fields for breaking-change type, plus recommendation for subtype field.", + "expectations": [ + "Flags missing impact field as schema error for breaking-change", + "Flags missing action field as schema error for breaking-change", + "Notes subtype field is strongly recommended for breaking-change", + "Does not suggest content (review only reports issues)", + "Proper markdown format with clear error descriptions" + ] + }, + { + "id": 7, + "prompt": "Review changelog with type-title alignment issues: Type 'bug-fix' but title 'Improve dashboard loading performance' suggesting enhancement rather than bug fix.", + "expected_output": "Quality warnings section identifying type-title alignment mismatch with explanation of why bug-fix type doesn't match improvement-focused title.", + "expectations": [ + "Detects type-title alignment mismatch in Quality warnings", + "Explains bug-fix type expects Fix/Resolve/Correct verbs", + "Notes title uses Improve suggesting enhancement type", + "Warns about potential misclassification", + "Does not provide fixes (review mode only reports)" + ] + }, + { + "id": 8, + "prompt": "Review changelog with overly technical title: 'Fix splitValue nullability coercion when constructing ColorSeries objects in ES|QL chart renderer'", + "expected_output": "Quality warnings section flagging technical content issues: implementation-focused title with class names and technical jargon without user context.", + "expectations": [ + "Flags title as too technical in Quality warnings section", + "Identifies class names and implementation details without user context", + "Warns title focuses on code changes rather than user-visible symptoms", + "Notes users can't determine impact from technical jargon", + "Suggests rewrite needed to focus on user experience" + ] + }, + { + "id": 9, + "prompt": "Review changelog with content quality issues: Title 'feat: Bug fixes', Description 'See PR', areas field as string instead of array.", + "expected_output": "Quality warnings section identifying development prefixes, vague title, low-value description, and schema error for incorrect areas field type.", + "expectations": [ + "Flags 'feat:' development prefix in Quality warnings", + "Identifies 'Bug fixes' as too vague", + "Flags 'See PR' as low-value description", + "Reports areas field type error in Schema errors section", + "Uses systematic pattern recognition for title cleanup issues" + ] + }, + { + "id": 10, + "prompt": "Review changelog with YAML formatting issues: unquoted description containing colons, brackets, and plain text code identifiers like validateToken method without backticks.", + "expected_output": "Formatting warnings section identifying YAML quoting needs and missing code formatting with specific examples of what needs backticks.", + "expectations": [ + "Flags unquoted text with colons in Formatting warnings", + "Identifies missing backticks around method names and code identifiers", + "Explains YAML parsing risks from special characters", + "Notes inconsistent code formatting affects readability", + "Provides specific examples of what needs formatting fixes" + ] + }, + { + "id": 11, + "prompt": "Review changelog with invalid area 'Dashboards' when repository config shows valid areas are 'Dashboards and visualizations' (repository-aware validation).", + "expected_output": "Quality warnings section showing area validation against repository configuration, noting the area doesn't match canonical list from docs/changelog.yml.", + "expectations": [ + "Validates areas against repository-specific configuration", + "Flags area mismatch in Quality warnings section", + "References successful repository config loading", + "Notes case sensitivity in area validation", + "Distinguishes repository-aware vs generic validation" + ] + }, + { + "id": 12, + "prompt": "Review directory ./mixed-changelogs/ containing files with various issues: perfect changelog, schema errors, quality problems, and formatting issues from different product types.", + "expected_output": "Multiple file reviews with individual sections per file, overall summary showing counts of files processed and files with issues, handling diverse product contexts.", + "expectations": [ + "Processes all yaml/yml files in directory", + "Provides individual review section for each file", + "Shows different issue types across files", + "Summarizes total files processed and files needing attention", + "Handles different product types and area taxonomies", + "Recommends docs-fix-changelog for files with warnings" + ] + }, + { + "id": 13, "prompt": "Review changelog with feature prefix pattern: title: 'File upload: Fixes disabling data view creation based on user capabilities'", "expected_output": "Flags feature/app prefix pattern as title standardization issue requiring contextual integration", "expectations": [ @@ -46,7 +156,7 @@ ] }, { - "id": 5, + "id": 14, "prompt": "Review changelog with technical colon usage: title: 'Fix bug with Auth header needing \"Authorization: Bearer \"'", "expected_output": "Correctly does NOT flag technical term as feature prefix, focuses on other potential issues", "expectations": [ @@ -57,7 +167,7 @@ ] }, { - "id": 6, + "id": 15, "prompt": "Review changelog with multiple title issues including feature prefix: title: 'Dashboard: fixes widget loading bugs'", "expected_output": "Flags both feature prefix pattern AND other title issues (verb form, capitalization)", "expectations": [ @@ -68,4 +178,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/skills/changelogs/test-samples/mixed-changelogs/formatting-problems.yaml b/skills/changelogs/test-samples/mixed-changelogs/formatting-problems.yaml new file mode 100644 index 0000000..d0403e8 --- /dev/null +++ b/skills/changelogs/test-samples/mixed-changelogs/formatting-problems.yaml @@ -0,0 +1,10 @@ +prs: +- https://github.com/elastic/logstash/pull/18045 +type: enhancement +products: +- product: logstash + target: 9.2.0 +areas: +- Input plugins +title: Update input configuration: now supports SSL parameters +description: The configureSSL method now handles certificate validation better and the parseConnectionString function supports new options. \ No newline at end of file diff --git a/skills/changelogs/test-samples/mixed-changelogs/good-kibana.yaml b/skills/changelogs/test-samples/mixed-changelogs/good-kibana.yaml new file mode 100644 index 0000000..621d39e --- /dev/null +++ b/skills/changelogs/test-samples/mixed-changelogs/good-kibana.yaml @@ -0,0 +1,10 @@ +prs: +- https://github.com/elastic/kibana/pull/266184 +type: feature +products: +- product: cloud-serverless + target: 2026-05-12 +areas: +- Dashboards and visualizations +title: "Add real-time collaboration features for dashboard editing" +description: "Multiple users can now edit dashboards simultaneously with live cursor tracking, conflict resolution, and automatic saving of collaborative changes." \ No newline at end of file diff --git a/skills/changelogs/test-samples/mixed-changelogs/problematic-elasticsearch.yaml b/skills/changelogs/test-samples/mixed-changelogs/problematic-elasticsearch.yaml new file mode 100644 index 0000000..fc2514e --- /dev/null +++ b/skills/changelogs/test-samples/mixed-changelogs/problematic-elasticsearch.yaml @@ -0,0 +1,9 @@ +prs: +- https://github.com/elastic/elasticsearch/pull/130089 +# Missing type field - schema error +products: +- product: elasticsearch + target: 9.2.0 +areas: "ES|QL and Search" # Should be array, not string - schema error +title: fix: Performance improvements # Development prefix + vague title - quality issues +description: Various optimizations # Vague description - quality issue \ No newline at end of file diff --git a/skills/changelogs/test-samples/perfect-changelog.yaml b/skills/changelogs/test-samples/perfect-changelog.yaml new file mode 100644 index 0000000..71f6adc --- /dev/null +++ b/skills/changelogs/test-samples/perfect-changelog.yaml @@ -0,0 +1,13 @@ +prs: +- https://github.com/elastic/kibana/pull/266184 +type: feature +products: +- product: cloud-serverless + target: 2026-05-12 +areas: +- Dashboards and visualizations +title: "Add custom visualization builder for advanced chart configurations" +description: >- + Users can now create custom visualizations using the new advanced chart builder. + This feature provides drag-and-drop interface for complex data relationships, + custom color schemes, and interactive filtering options that enhance dashboard functionality. \ No newline at end of file