From 18e9f7db71862ef7812dda1266c2004a3c639ac7 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Thu, 21 Aug 2025 15:07:17 -0500 Subject: [PATCH 1/7] feat: add automated documentation sync reminders and template improvements - Add GitHub Actions workflow to detect README changes and create sync reminders - Update README_TEMPLATE.md with documentation sync instructions - Add validate_readme.py script for template compliance checking - Update CONTRIBUTING.md with comprehensive documentation sync process guide The workflow automatically: - Detects plugin README changes on master branch commits - Creates commit comments with pre-filled sync request links to docs-v2 - Provides clear instructions for the sync process - Requires no secrets or cross-repository authentication Updated documentation includes: - Complete sync workflow instructions - Template compliance requirements - Automated sync process explanation - Manual sync alternatives --- .github/workflows/remind-sync-docs.yml | 122 +++++++++ CONTRIBUTING.md | 58 ++++ README_TEMPLATE.md | 350 +++++++++++++++++++++++++ validate_readme.py | 301 +++++++++++++++++++++ 4 files changed, 831 insertions(+) create mode 100644 .github/workflows/remind-sync-docs.yml create mode 100644 README_TEMPLATE.md create mode 100644 validate_readme.py diff --git a/.github/workflows/remind-sync-docs.yml b/.github/workflows/remind-sync-docs.yml new file mode 100644 index 0000000..90e8e99 --- /dev/null +++ b/.github/workflows/remind-sync-docs.yml @@ -0,0 +1,122 @@ +name: Remind to Sync Documentation + +on: + push: + branches: [master] + paths: + - 'influxdata/**/README.md' + +permissions: + contents: read + +jobs: + remind-sync: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 # Need previous commit to detect changes + + - name: Detect changed plugins + id: detect-changes + run: | + # Get list of changed README files + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep '^influxdata/.*/README\.md$' | head -10) + + if [[ -z "$CHANGED_FILES" ]]; then + echo "No plugin README files changed" + echo "changed_plugins=" >> $GITHUB_OUTPUT + echo "has_changes=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Changed files:" + echo "$CHANGED_FILES" + + # Extract plugin names from file paths + PLUGIN_NAMES="" + while IFS= read -r file; do + if [[ -n "$file" ]]; then + # Extract plugin name from path: influxdata/plugin_name/README.md -> plugin_name + PLUGIN_NAME=$(echo "$file" | sed 's|influxdata/||' | sed 's|/README\.md||') + if [[ -n "$PLUGIN_NAMES" ]]; then + PLUGIN_NAMES="$PLUGIN_NAMES, $PLUGIN_NAME" + else + PLUGIN_NAMES="$PLUGIN_NAME" + fi + fi + done <<< "$CHANGED_FILES" + + echo "Changed plugins: $PLUGIN_NAMES" + echo "changed_plugins=$PLUGIN_NAMES" >> $GITHUB_OUTPUT + echo "has_changes=true" >> $GITHUB_OUTPUT + + - name: Create sync reminder comment + if: steps.detect-changes.outputs.has_changes == 'true' + uses: actions/github-script@v7 + with: + script: | + const changedPlugins = '${{ steps.detect-changes.outputs.changed_plugins }}'; + const commitSha = context.sha; + const shortSha = commitSha.substring(0, 7); + + // Build the GitHub issue URL with pre-filled parameters + const baseUrl = 'https://github.com/influxdata/docs-v2/issues/new'; + const template = 'sync-plugin-docs.yml'; + const title = encodeURIComponent(`Sync plugin docs: ${changedPlugins}`); + const plugins = encodeURIComponent(changedPlugins); + const sourceCommit = encodeURIComponent(commitSha); + + const issueUrl = `${baseUrl}?template=${template}&title=${title}&plugins=${plugins}&source_commit=${sourceCommit}`; + + // Create the comment body + const commentBody = `šŸ“š **Plugin documentation updated!** + + The following plugin READMEs were changed in this commit: + **${changedPlugins}** + + ## Next Steps + + To sync these changes to the InfluxDB documentation site: + + ### šŸš€ [**Click here to create sync request**](${issueUrl}) + + This will open a pre-filled issue in docs-v2 that will automatically trigger the sync workflow. + + ### What the sync process does: + 1. āœ… Validates your plugin READMEs against template requirements + 2. šŸ”„ Transforms content for docs-v2 compatibility (adds Hugo shortcodes, fixes links) + 3. šŸ–¼ļø Generates screenshots of the plugin documentation pages + 4. šŸ“ Creates a pull request in docs-v2 ready for review + + ### Before syncing: + - Ensure your README follows the [README_TEMPLATE.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README_TEMPLATE.md) structure + - Include proper emoji metadata (āš” triggers, šŸ·ļø tags, šŸ”§ compatibility) + - Verify all required sections are present and complete + + --- + *Commit: ${shortSha} | [View workflow](https://github.com/influxdata/docs-v2/blob/master/helper-scripts/influxdb3-plugins/README.md)*`; + + // Create commit comment + await github.rest.repos.createCommitComment({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: commitSha, + body: commentBody + }); + + console.log(`Created sync reminder for plugins: ${changedPlugins}`); + console.log(`Issue URL: ${issueUrl}`); + + - name: Log workflow completion + if: steps.detect-changes.outputs.has_changes == 'true' + run: | + echo "āœ… Sync reminder created for plugins: ${{ steps.detect-changes.outputs.changed_plugins }}" + echo "šŸ”— Users can click the link in the commit comment to trigger docs sync" + + - name: No changes detected + if: steps.detect-changes.outputs.has_changes == 'false' + run: | + echo "ā„¹ļø No plugin README files were changed in this commit" \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0d093a4..0f0940e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -489,6 +489,51 @@ raise ValueError( ) ``` +## Documentation Sync Process + +### Syncing Plugin Documentation to docs-v2 + +When you update plugin READMEs in this repository, they need to be synchronized to the InfluxDB documentation site (docs-v2) to appear on docs.influxdata.com. + +#### Automated Sync Workflow + +1. **Make your changes** - Update plugin README files following the template structure +2. **Commit and push** - Push your changes to the master branch +3. **Check for reminder** - A workflow will automatically detect README changes and create a commit comment with a sync link +4. **Click sync link** - The comment includes a pre-filled link to create a sync request in docs-v2 +5. **Submit request** - This automatically triggers validation, transformation, and PR creation +6. **Review PR** - The resulting PR in docs-v2 includes screenshots and change summaries + +#### Manual Sync (Alternative) + +You can also manually trigger synchronization: + +1. **Navigate to sync form**: [Create sync request](https://github.com/influxdata/docs-v2/issues/new?template=sync-plugin-docs.yml) +2. **Fill in details** - Specify plugin names and source commit +3. **Submit** - The automation workflow handles the rest + +#### Sync Requirements + +Before syncing, ensure your README: + +- āœ… Follows the [README_TEMPLATE.md](README_TEMPLATE.md) structure +- āœ… Includes proper emoji metadata (`āš”` triggers, `šŸ·ļø` tags, `šŸ”§` compatibility) +- āœ… Has all required sections with proper formatting +- āœ… Contains working examples with expected output +- āœ… Passes validation (`python validate_readme.py`) + +#### What Gets Transformed + +During sync, your README content is automatically transformed for docs-v2: + +- **Emoji metadata removed** (already in plugin JSON metadata) +- **Relative links converted** to GitHub URLs +- **Product references enhanced** with Hugo shortcodes (`{{% product-name %}}`) +- **Logging section added** with standard content +- **Support sections updated** with docs-v2 format + +Your original README remains unchanged - these transformations only apply to the docs-v2 copy. + ## Commit Message Format ### Use conventional commits @@ -509,4 +554,17 @@ raise ValueError( - `test`: Test changes - `chore`: Maintenance tasks +### Documentation Sync Messages + +When making documentation-focused commits, use clear messages that describe what changed: + +```bash +# Good commit messages for docs sync +docs(basic_transformation): update configuration parameters and examples +feat(downsampler): add support for custom aggregation functions +fix(notifier): correct email configuration example + +# These will trigger sync reminders automatically +``` + *These standards are extracted from the [InfluxData Documentation guidelines](https://github.com/influxdata/docs-v2/blob/master/CONTRIBUTING.md).* diff --git a/README_TEMPLATE.md b/README_TEMPLATE.md new file mode 100644 index 0000000..de8777f --- /dev/null +++ b/README_TEMPLATE.md @@ -0,0 +1,350 @@ +# Plugin Name + +āš” scheduled, data-write, http šŸ·ļø tag1, tag2, tag3 šŸ”§ InfluxDB 3 Core, InfluxDB 3 Enterprise + +## Description + +Brief description of what the plugin does and its primary use case. Include the trigger types supported (write, scheduled, HTTP) and main functionality. Mention any special features or capabilities that distinguish this plugin. Add a fourth sentence if needed for additional context. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. Some plugins support TOML configuration files, which can be specified using the plugin's `config_file_path` parameter. + +If a plugin supports multiple trigger specifications, some parameters may depend on the trigger specification that you use. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `parameter_name` | string | required | Description of the parameter | +| `another_param` | integer | required | Description with any constraints or requirements | + +### Optional parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `optional_param` | boolean | "false" | Description of optional parameter | +| `timeout` | integer | 30 | Connection timeout in seconds | + +### [Category] parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `category_param` | string | "default" | Parameters grouped by functionality | + +### TOML configuration + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | + +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. + +#### Example TOML configurations + +- [plugin_config_scheduler.toml](plugin_config_scheduler.toml) - for scheduled triggers +- [plugin_config_data_writes.toml](plugin_config_data_writes.toml) - for data write triggers + +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](/README.md). + +## [Special Requirements Section] + + + +### Data requirements + +The plugin requires [specific data format or schema requirements]. + +### Software requirements + +- **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled +- **Python packages**: + - `package_name` (for specific functionality) + +## Installation steps + +1. Start InfluxDB 3 with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` + +2. Install required Python packages (if any): + + ```bash + influxdb3 install package package_name + ``` + +## Trigger setup + +### Scheduled trigger + +Run the plugin periodically on historical data: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "every:1h" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + scheduled_trigger_name +``` + +### Data write trigger + +Process data as it's written: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "all_tables" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + write_trigger_name +``` + +### HTTP trigger + +Process data via HTTP requests: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "request:endpoint" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + http_trigger_name +``` + +## Example usage + +### Example 1: [Use case name] + +[Description of what this example demonstrates] + +```bash +# Create the trigger +influxdb3 create trigger \ + --database weather \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "every:30m" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + example_trigger + +# Write test data +influxdb3 write \ + --database weather \ + "measurement,tag=value field=22.5" + +# Query results (after trigger runs) +influxdb3 query \ + --database weather \ + "SELECT * FROM result_measurement" +``` + +### Expected output + +``` +tag | field | time +----|-------|----- +value | 22.5 | 2024-01-01T00:00:00Z +``` + +**Transformation details:** +- Before: `field=22.5` (original value) +- After: `field=22.5` (processed value with description of changes) + +### Example 2: [Another use case] + +[Description of what this example demonstrates] + +```bash +# Create trigger with different configuration +influxdb3 create trigger \ + --database sensors \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "all_tables" \ + --trigger-arguments 'parameter_name=different_value,optional_param=true' \ + another_trigger + +# Write data with specific format +influxdb3 write \ + --database sensors \ + "raw_data,device=sensor1 value1=20.1,value2=45.2" + +# Query processed data +influxdb3 query \ + --database sensors \ + "SELECT * FROM processed_data" +``` + +### Expected output + +``` +device | value1 | value2 | time +-------|--------|--------|----- +sensor1 | 20.1 | 45.2 | 2024-01-01T00:00:00Z +``` + +**Processing details:** +- Before: `value1=20.1`, `value2=45.2` +- After: [Description of any transformations applied] + +### Example 3: [Complex scenario] + +[Add more examples as needed to demonstrate different features] + +## Code overview + +### Files + +- `plugin_name.py`: Main plugin code containing handlers for trigger types +- `plugin_config_scheduler.toml`: Example TOML configuration for scheduled triggers +- `plugin_config_data_writes.toml`: Example TOML configuration for data write triggers + +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` +Handles scheduled trigger execution. Queries historical data within the specified window and applies processing logic. + +Key operations: +1. Parses configuration from arguments +2. Queries source data with filters +3. Applies processing logic +4. Writes results to target measurement + +#### `process_writes(influxdb3_local, table_batches, args)` +Handles real-time data processing during writes. Processes incoming data batches and applies transformations before writing. + +Key operations: +1. Filters relevant table batches +2. Applies processing to each row +3. Writes to target measurement immediately + +#### `process_http_request(influxdb3_local, request_body, args)` +Handles HTTP-triggered processing. Processes data sent via HTTP requests. + +Key operations: +1. Parses request body +2. Validates input data +3. Applies processing logic +4. Returns response + +### Key logic + +1. **Data Validation**: Checks input data format and required fields +2. **Processing**: Applies the main plugin logic to transform/analyze data +3. **Output Generation**: Formats results and metadata +4. **Error Handling**: Manages exceptions and provides meaningful error messages + +### Plugin Architecture + +``` +Plugin Module +ā”œā”€ā”€ process_scheduled_call() # Scheduled trigger handler +ā”œā”€ā”€ process_writes() # Data write trigger handler +ā”œā”€ā”€ process_http_request() # HTTP trigger handler +ā”œā”€ā”€ validate_config() # Configuration validation +ā”œā”€ā”€ apply_processing() # Core processing logic +ā””ā”€ā”€ helper_functions() # Utility functions +``` + +## Troubleshooting + +### Common issues + +#### Issue: "Configuration parameter missing" +**Solution**: Check that all required parameters are provided in the trigger arguments. Verify parameter names match exactly (case-sensitive). + +#### Issue: "Permission denied" errors +**Solution**: Ensure the plugin file has execute permissions: +```bash +chmod +x ~/.plugins/plugin_name.py +``` + +#### Issue: "Module not found" error +**Solution**: Install required Python packages: +```bash +influxdb3 install package package_name +``` + +#### Issue: No data in target measurement +**Solution**: +1. Check that source measurement contains data +2. Verify trigger is enabled and running +3. Check logs for errors: + ```bash + influxdb3 query \ + --database _internal \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" + ``` + +### Debugging tips + +1. **Enable debug logging**: Add `debug=true` to trigger arguments +2. **Use dry run mode**: Set `dry_run=true` to test without writing data +3. **Check field names**: Use `SHOW FIELD KEYS FROM measurement` to verify field names +4. **Test with small windows**: Use short time windows for testing (e.g., `window=1h`) +5. **Monitor resource usage**: Check CPU and memory usage during processing + +### Performance considerations + +- Processing large datasets may require increased memory +- Use filters to process only relevant data +- Batch size affects memory usage and processing speed +- Consider using specific_fields to limit processing scope +- Cache frequently accessed data when possible + +## Questions/Comments + +For questions or comments about this plugin, please open an issue in the [influxdb3_plugins repository](https://github.com/influxdata/influxdb3_plugins/issues). + +--- + +## Documentation Sync + +After making changes to this README, sync to the documentation site: + +1. **Commit your changes** to the influxdb3_plugins repository +2. **Look for the sync reminder** - A comment will appear on your commit with a sync link +3. **Click the link** - This opens a pre-filled form to trigger the docs-v2 sync +4. **Submit the sync request** - A workflow will validate, transform, and create a PR + +The documentation site will be automatically updated with your changes after review. + +--- + +## Template Usage Notes + +When using this template: + +1. Replace `Plugin Name` with the actual plugin name +2. Update emoji metadata with appropriate trigger types and tags +3. Fill in all parameter tables with actual configuration options +4. Provide real, working examples with expected output +5. Include actual function names and signatures +6. Add plugin-specific troubleshooting scenarios +7. Remove any sections that don't apply to your plugin +8. Remove this "Template Usage Notes" section from the final README + +### Section Guidelines + +- **Description**: 2-4 sentences, be specific about capabilities +- **Configuration**: Group parameters logically, mark required clearly +- **Examples**: At least 2 complete, working examples +- **Expected output**: Show actual output format +- **Troubleshooting**: Include plugin-specific issues and solutions +- **Code overview**: Document main functions and logic flow \ No newline at end of file diff --git a/validate_readme.py b/validate_readme.py new file mode 100644 index 0000000..3c2fadf --- /dev/null +++ b/validate_readme.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +""" +Validates plugin README files against the standard template. +Ensures consistency across all plugin documentation. +""" + +import sys +import re +from pathlib import Path +from typing import List, Tuple + +# Required sections in order +REQUIRED_SECTIONS = [ + ("# ", "Title with plugin name"), + ("## Description", "Plugin description"), + ("## Configuration", "Configuration overview"), + ("### Plugin metadata", "Metadata description"), + ("### Required parameters", "Required parameters table"), + ("## Installation steps", "Installation instructions"), + ("## Trigger setup", "Trigger configuration examples"), + ("## Example usage", "Usage examples"), + ("## Code overview", "Code structure description"), + ("## Troubleshooting", "Common issues and solutions"), + ("## Questions/Comments", "Support information") +] + +# Optional but recommended sections +OPTIONAL_SECTIONS = [ + "### Optional parameters", + "### TOML configuration", + "### Data requirements", + "### Software requirements", + "### Schema requirements", + "### Debugging tips", + "### Performance considerations" +] + +def validate_emoji_metadata(content: str) -> List[str]: + """Validate the emoji metadata line.""" + errors = [] + + # Check for emoji metadata pattern + metadata_pattern = r'^āš”\s+[\w\-,\s]+\s+šŸ·ļø\s+[\w\-,\s]+\s+šŸ”§\s+InfluxDB 3' + if not re.search(metadata_pattern, content, re.MULTILINE): + errors.append("Missing or invalid emoji metadata line (should have āš” trigger types šŸ·ļø tags šŸ”§ compatibility)") + + return errors + +def validate_sections(content: str) -> List[str]: + """Validate required sections are present and in order.""" + errors = [] + lines = content.split('\n') + + # Track section positions + section_positions = {} + for i, line in enumerate(lines): + for section, description in REQUIRED_SECTIONS: + if line.startswith(section) and section not in section_positions: + # Special handling for title (should contain actual plugin name) + if section == "# " and not line.startswith("# Plugin Name"): + section_positions[section] = i + elif section != "# ": + section_positions[section] = i + + # Check all required sections are present + for section, description in REQUIRED_SECTIONS: + if section not in section_positions: + errors.append(f"Missing required section: '{section.strip()}' - {description}") + + # Check sections are in correct order + if len(section_positions) == len(REQUIRED_SECTIONS): + positions = list(section_positions.values()) + if positions != sorted(positions): + errors.append("Sections are not in the correct order (see template for proper ordering)") + + return errors + +def validate_parameter_tables(content: str) -> List[str]: + """Validate parameter table formatting.""" + errors = [] + + # Check for parameter table headers + if '| Parameter | Type | Default | Description |' not in content: + errors.append("No properly formatted parameter tables found (should have Parameter | Type | Default | Description columns)") + + # Check for required parameters section + if '### Required parameters' in content: + section_start = content.index('### Required parameters') + section_end = content.find('\n###', section_start + 1) + if section_end == -1: + section_end = content.find('\n##', section_start + 1) + + section_content = content[section_start:section_end] if section_end != -1 else content[section_start:] + + if 'required' not in section_content.lower(): + errors.append("Required parameters section should indicate which parameters are required") + + return errors + +def validate_examples(content: str) -> List[str]: + """Validate code examples and expected output.""" + errors = [] + + # Check for bash code examples + bash_examples = re.findall(r'```bash(.*?)```', content, re.DOTALL) + if len(bash_examples) < 2: + errors.append(f"Should have at least 2 bash code examples (found {len(bash_examples)})") + + # Check for influxdb3 commands in examples + has_create_trigger = any('influxdb3 create trigger' in ex for ex in bash_examples) + has_write_data = any('influxdb3 write' in ex for ex in bash_examples) + has_query = any('influxdb3 query' in ex for ex in bash_examples) + + if not has_create_trigger: + errors.append("Examples should include 'influxdb3 create trigger' command") + if not has_write_data: + errors.append("Examples should include 'influxdb3 write' command for test data") + if not has_query: + errors.append("Examples should include 'influxdb3 query' command to verify results") + + # Check for expected output + expected_output_count = content.count('### Expected output') + content.count('**Expected output') + if expected_output_count < 1: + errors.append("Should include at least one 'Expected output' section in examples") + + return errors + +def validate_links(content: str, plugin_path: Path) -> List[str]: + """Validate internal links and references.""" + errors = [] + + # Check for TOML file references if TOML configuration is mentioned + if '### TOML configuration' in content: + toml_links = re.findall(r'\[([^\]]+\.toml)\]\(([^)]+)\)', content) + plugin_dir = plugin_path.parent + + for link_text, link_path in toml_links: + # Check if it's a relative link (not starting with http) + if not link_path.startswith('http'): + toml_file = plugin_dir / link_path + if not toml_file.exists(): + errors.append(f"Referenced TOML file not found: {link_path}") + + # Check for influxdb3_plugins README reference + if '/README.md' in content and 'influxdb3_plugins/README.md' not in content: + errors.append("Link to main README should reference 'influxdb3_plugins/README.md'") + + return errors + +def validate_troubleshooting(content: str) -> List[str]: + """Validate troubleshooting section content.""" + errors = [] + + if '## Troubleshooting' in content: + section_start = content.index('## Troubleshooting') + section_end = content.find('\n##', section_start + 1) + section_content = content[section_start:section_end] if section_end != -1 else content[section_start:] + + # Check for common subsections + if '### Common issues' not in section_content: + errors.append("Troubleshooting should include 'Common issues' subsection") + + # Check for issue/solution pattern + issue_count = section_content.count('#### Issue:') + section_content.count('**Issue:') + solution_count = section_content.count('**Solution:') + section_content.count('Solution:') + + if issue_count < 2: + errors.append("Troubleshooting should include at least 2 documented issues") + if issue_count > solution_count: + errors.append("Each troubleshooting issue should have a corresponding solution") + + return errors + +def validate_code_overview(content: str) -> List[str]: + """Validate code overview section.""" + errors = [] + + if '## Code overview' in content: + section_start = content.index('## Code overview') + section_end = content.find('\n##', section_start + 1) + section_content = content[section_start:section_end] if section_end != -1 else content[section_start:] + + # Check for required subsections + if '### Files' not in section_content: + errors.append("Code overview should include 'Files' subsection") + if '### Main functions' not in section_content and '### Key functions' not in section_content: + errors.append("Code overview should include 'Main functions' or 'Key functions' subsection") + + # Check for function documentation + if 'def ' not in section_content and not re.search(r'`\w+\(.*?\)`', section_content): + errors.append("Code overview should document main functions with their signatures") + + return errors + +def format_validation_result(readme_path: Path, errors: List[str], warnings: List[str]) -> str: + """Format validation results for display.""" + result = [] + + if not errors and not warnings: + result.append(f"āœ… {readme_path}") + else: + result.append(f"\n{'āŒ' if errors else 'āš ļø'} {readme_path}:") + + if errors: + result.append(" Errors:") + for error in errors: + result.append(f" - {error}") + + if warnings: + result.append(" Warnings:") + for warning in warnings: + result.append(f" - {warning}") + + return '\n'.join(result) + +def validate_readme(readme_path: Path) -> Tuple[List[str], List[str]]: + """ + Validate a single README file. + Returns tuple of (errors, warnings). + """ + try: + with open(readme_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception as e: + return [f"Could not read file: {e}"], [] + + errors = [] + warnings = [] + + # Run all validations + errors.extend(validate_emoji_metadata(content)) + errors.extend(validate_sections(content)) + errors.extend(validate_parameter_tables(content)) + errors.extend(validate_examples(content)) + errors.extend(validate_links(content, readme_path)) + errors.extend(validate_troubleshooting(content)) + errors.extend(validate_code_overview(content)) + + # Check for optional but recommended sections + for section in OPTIONAL_SECTIONS: + if section not in content and section not in ["### Debugging tips", "### Performance considerations"]: + warnings.append(f"Consider adding '{section}' section") + + # Check for template remnants + if 'Plugin Name' in content and '# Plugin Name' in content: + errors.append("README still contains template placeholder 'Plugin Name'") + if 'Template Usage Notes' in content: + errors.append("README still contains 'Template Usage Notes' section (should be removed)") + + return errors, warnings + +def main(): + """Main validation function.""" + # Find all plugin READMEs + influxdata_dir = Path('influxdata') + + if not influxdata_dir.exists(): + print("āŒ Error: 'influxdata' directory not found. Run this script from the influxdb3_plugins root directory.") + sys.exit(1) + + readme_files = list(influxdata_dir.glob('*/README.md')) + + if not readme_files: + print("āŒ No README files found in influxdata/ subdirectories") + sys.exit(1) + + print(f"Validating {len(readme_files)} plugin README files...\n") + + all_valid = True + error_count = 0 + warning_count = 0 + + for readme_path in sorted(readme_files): + errors, warnings = validate_readme(readme_path) + + if errors: + all_valid = False + error_count += len(errors) + warning_count += len(warnings) + + print(format_validation_result(readme_path, errors, warnings)) + + # Print summary + print("\n" + "=" * 60) + print("VALIDATION SUMMARY") + print("=" * 60) + print(f"Total files validated: {len(readme_files)}") + print(f"Errors found: {error_count}") + print(f"Warnings found: {warning_count}") + + if all_valid: + print("\nāœ… All README files are valid!") + sys.exit(0) + else: + print(f"\nāŒ Validation failed with {error_count} error(s)") + print("\nPlease fix the errors above and ensure all READMEs follow the template.") + print("See README_TEMPLATE.md for the correct structure.") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file From cf7a557fd5993a6eb56b69d87c41d9bc68813176 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Thu, 21 Aug 2025 16:42:12 -0500 Subject: [PATCH 2/7] docs: Improve validations: add validate_readme --help, fix magic numbers, move to scripts directory:- Moves validate_readme.py to /scripts - Replaces magic numbers with variables - Adds support for --help and --list flags- Updates contributing doc --- CONTRIBUTING.md | 2 +- .../validate_readme.py | 203 +++++++++++++++--- 2 files changed, 172 insertions(+), 33 deletions(-) rename validate_readme.py => scripts/validate_readme.py (61%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f0940e..8b516bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -520,7 +520,7 @@ Before syncing, ensure your README: - āœ… Includes proper emoji metadata (`āš”` triggers, `šŸ·ļø` tags, `šŸ”§` compatibility) - āœ… Has all required sections with proper formatting - āœ… Contains working examples with expected output -- āœ… Passes validation (`python validate_readme.py`) +- āœ… Passes validation (`python scripts/validate_readme.py`) #### What Gets Transformed diff --git a/validate_readme.py b/scripts/validate_readme.py similarity index 61% rename from validate_readme.py rename to scripts/validate_readme.py index 3c2fadf..4acbe84 100644 --- a/validate_readme.py +++ b/scripts/validate_readme.py @@ -6,9 +6,24 @@ import sys import re +import argparse from pathlib import Path from typing import List, Tuple +# Validation thresholds and limits +MINIMUM_BASH_EXAMPLES = 2 +MINIMUM_EXPECTED_OUTPUT_SECTIONS = 1 +MINIMUM_TROUBLESHOOTING_ISSUES = 2 +SUMMARY_SEPARATOR_LENGTH = 60 + +# Section search offsets +SECTION_SEARCH_OFFSET = 1 +SECTION_NOT_FOUND = -1 + +# Exit codes +EXIT_SUCCESS = 0 +EXIT_ERROR = 1 + # Required sections in order REQUIRED_SECTIONS = [ ("# ", "Title with plugin name"), @@ -86,11 +101,11 @@ def validate_parameter_tables(content: str) -> List[str]: # Check for required parameters section if '### Required parameters' in content: section_start = content.index('### Required parameters') - section_end = content.find('\n###', section_start + 1) - if section_end == -1: - section_end = content.find('\n##', section_start + 1) + section_end = content.find('\n###', section_start + SECTION_SEARCH_OFFSET) + if section_end == SECTION_NOT_FOUND: + section_end = content.find('\n##', section_start + SECTION_SEARCH_OFFSET) - section_content = content[section_start:section_end] if section_end != -1 else content[section_start:] + section_content = content[section_start:section_end] if section_end != SECTION_NOT_FOUND else content[section_start:] if 'required' not in section_content.lower(): errors.append("Required parameters section should indicate which parameters are required") @@ -103,8 +118,8 @@ def validate_examples(content: str) -> List[str]: # Check for bash code examples bash_examples = re.findall(r'```bash(.*?)```', content, re.DOTALL) - if len(bash_examples) < 2: - errors.append(f"Should have at least 2 bash code examples (found {len(bash_examples)})") + if len(bash_examples) < MINIMUM_BASH_EXAMPLES: + errors.append(f"Should have at least {MINIMUM_BASH_EXAMPLES} bash code examples (found {len(bash_examples)})") # Check for influxdb3 commands in examples has_create_trigger = any('influxdb3 create trigger' in ex for ex in bash_examples) @@ -120,8 +135,8 @@ def validate_examples(content: str) -> List[str]: # Check for expected output expected_output_count = content.count('### Expected output') + content.count('**Expected output') - if expected_output_count < 1: - errors.append("Should include at least one 'Expected output' section in examples") + if expected_output_count < MINIMUM_EXPECTED_OUTPUT_SECTIONS: + errors.append(f"Should include at least {MINIMUM_EXPECTED_OUTPUT_SECTIONS} 'Expected output' section in examples") return errors @@ -153,8 +168,8 @@ def validate_troubleshooting(content: str) -> List[str]: if '## Troubleshooting' in content: section_start = content.index('## Troubleshooting') - section_end = content.find('\n##', section_start + 1) - section_content = content[section_start:section_end] if section_end != -1 else content[section_start:] + section_end = content.find('\n##', section_start + SECTION_SEARCH_OFFSET) + section_content = content[section_start:section_end] if section_end != SECTION_NOT_FOUND else content[section_start:] # Check for common subsections if '### Common issues' not in section_content: @@ -164,8 +179,8 @@ def validate_troubleshooting(content: str) -> List[str]: issue_count = section_content.count('#### Issue:') + section_content.count('**Issue:') solution_count = section_content.count('**Solution:') + section_content.count('Solution:') - if issue_count < 2: - errors.append("Troubleshooting should include at least 2 documented issues") + if issue_count < MINIMUM_TROUBLESHOOTING_ISSUES: + errors.append(f"Troubleshooting should include at least {MINIMUM_TROUBLESHOOTING_ISSUES} documented issues") if issue_count > solution_count: errors.append("Each troubleshooting issue should have a corresponding solution") @@ -177,8 +192,8 @@ def validate_code_overview(content: str) -> List[str]: if '## Code overview' in content: section_start = content.index('## Code overview') - section_end = content.find('\n##', section_start + 1) - section_content = content[section_start:section_end] if section_end != -1 else content[section_start:] + section_end = content.find('\n##', section_start + SECTION_SEARCH_OFFSET) + section_content = content[section_start:section_end] if section_end != SECTION_NOT_FOUND else content[section_start:] # Check for required subsections if '### Files' not in section_content: @@ -249,22 +264,123 @@ def validate_readme(readme_path: Path) -> Tuple[List[str], List[str]]: return errors, warnings +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description='Validates plugin README files against the standard template.', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +Examples: + python scripts/validate_readme.py # Validate all plugins + python scripts/validate_readme.py --plugins basic_transformation,downsampler + python scripts/validate_readme.py --list # List available plugins + python scripts/validate_readme.py --quiet # Show only errors + +Validation Rules: + - Checks for required sections in correct order + - Validates emoji metadata format + - Ensures parameter tables are properly formatted + - Verifies code examples include required commands + - Validates troubleshooting content structure + ''' + ) + + parser.add_argument( + '--plugins', + type=str, + help='Comma-separated list of specific plugins to validate (e.g., "basic_transformation,downsampler")' + ) + + parser.add_argument( + '--list', + action='store_true', + help='List all available plugins and exit' + ) + + parser.add_argument( + '--quiet', + action='store_true', + help='Show only errors, suppress warnings and success messages' + ) + + parser.add_argument( + '--errors-only', + action='store_true', + help='Exit with success code even if warnings are found (only fail on errors)' + ) + + return parser.parse_args() + +def list_available_plugins(): + """List all available plugins and exit.""" + influxdata_dir = Path('influxdata') + + if not influxdata_dir.exists(): + print("āŒ Error: 'influxdata' directory not found. Run this script from the influxdb3_plugins root directory.") + sys.exit(EXIT_ERROR) + + readme_files = list(influxdata_dir.glob('*/README.md')) + + if not readme_files: + print("āŒ No plugins found in influxdata/ subdirectories") + sys.exit(EXIT_ERROR) + + print(f"Available plugins ({len(readme_files)} found):") + for readme_path in sorted(readme_files): + plugin_name = readme_path.parent.name + print(f" - {plugin_name}") + + sys.exit(EXIT_SUCCESS) + +def filter_plugins_by_name(readme_files: List[Path], plugin_names: str) -> List[Path]: + """Filter README files by specified plugin names.""" + requested_plugins = [name.strip() for name in plugin_names.split(',')] + filtered_files = [] + + for readme_path in readme_files: + plugin_name = readme_path.parent.name + if plugin_name in requested_plugins: + filtered_files.append(readme_path) + requested_plugins.remove(plugin_name) + + # Report any plugins that weren't found + if requested_plugins: + print(f"āš ļø Warning: The following plugins were not found: {', '.join(requested_plugins)}") + available_plugins = [f.parent.name for f in readme_files] + print(f"Available plugins: {', '.join(sorted(available_plugins))}") + + return filtered_files + def main(): """Main validation function.""" + args = parse_arguments() + + # Handle list option + if args.list: + list_available_plugins() + # Find all plugin READMEs influxdata_dir = Path('influxdata') if not influxdata_dir.exists(): print("āŒ Error: 'influxdata' directory not found. Run this script from the influxdb3_plugins root directory.") - sys.exit(1) + sys.exit(EXIT_ERROR) readme_files = list(influxdata_dir.glob('*/README.md')) if not readme_files: print("āŒ No README files found in influxdata/ subdirectories") - sys.exit(1) + sys.exit(EXIT_ERROR) - print(f"Validating {len(readme_files)} plugin README files...\n") + # Filter by specific plugins if requested + if args.plugins: + readme_files = filter_plugins_by_name(readme_files, args.plugins) + if not readme_files: + print("āŒ No matching plugins found") + sys.exit(EXIT_ERROR) + + if not args.quiet: + print(f"Validating {len(readme_files)} plugin README files...\n") all_valid = True error_count = 0 @@ -278,24 +394,47 @@ def main(): error_count += len(errors) warning_count += len(warnings) - print(format_validation_result(readme_path, errors, warnings)) + result = format_validation_result(readme_path, errors, warnings) + + # Apply quiet mode filtering + if args.quiet: + # Only show files with errors in quiet mode + if errors: + print(result) + else: + print(result) # Print summary - print("\n" + "=" * 60) - print("VALIDATION SUMMARY") - print("=" * 60) - print(f"Total files validated: {len(readme_files)}") - print(f"Errors found: {error_count}") - print(f"Warnings found: {warning_count}") - - if all_valid: - print("\nāœ… All README files are valid!") - sys.exit(0) + if not args.quiet: + print("\n" + "=" * SUMMARY_SEPARATOR_LENGTH) + print("VALIDATION SUMMARY") + print("=" * SUMMARY_SEPARATOR_LENGTH) + print(f"Total files validated: {len(readme_files)}") + print(f"Errors found: {error_count}") + print(f"Warnings found: {warning_count}") + + # Determine exit status + has_errors = error_count > 0 + has_warnings = warning_count > 0 + + if not has_errors and not has_warnings: + if not args.quiet: + print("\nāœ… All README files are valid!") + sys.exit(EXIT_SUCCESS) + elif not has_errors and has_warnings: + if not args.quiet: + print(f"\nāš ļø Validation completed with {warning_count} warning(s) but no errors") + # If --errors-only is specified, exit successfully even with warnings + exit_code = EXIT_SUCCESS if args.errors_only else EXIT_ERROR + sys.exit(exit_code) else: - print(f"\nāŒ Validation failed with {error_count} error(s)") - print("\nPlease fix the errors above and ensure all READMEs follow the template.") - print("See README_TEMPLATE.md for the correct structure.") - sys.exit(1) + if not args.quiet: + print(f"\nāŒ Validation failed with {error_count} error(s)") + if has_warnings: + print(f"Also found {warning_count} warning(s)") + print("\nPlease fix the errors above and ensure all READMEs follow the template.") + print("See README_TEMPLATE.md for the correct structure.") + sys.exit(EXIT_ERROR) if __name__ == "__main__": main() \ No newline at end of file From bf5cd1585408b2cf66687b205eb0d72c8d1cf6e4 Mon Sep 17 00:00:00 2001 From: meelahme Date: Wed, 8 Oct 2025 11:24:48 -0700 Subject: [PATCH 3/7] fix: update workflow to trigger on main branch instead of master --- .github/workflows/remind-sync-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/remind-sync-docs.yml b/.github/workflows/remind-sync-docs.yml index 90e8e99..4de28fd 100644 --- a/.github/workflows/remind-sync-docs.yml +++ b/.github/workflows/remind-sync-docs.yml @@ -2,7 +2,7 @@ name: Remind to Sync Documentation on: push: - branches: [master] + branches: [main] paths: - 'influxdata/**/README.md' From a4bd405aab238d686762dcc80861e63303128e17 Mon Sep 17 00:00:00 2001 From: meelahme Date: Wed, 8 Oct 2025 12:16:03 -0700 Subject: [PATCH 4/7] fix: improve validation and fix workflow branch trigger --- influxdata/basic_transformation/README.md | 5 ++--- scripts/validate_readme.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/influxdata/basic_transformation/README.md b/influxdata/basic_transformation/README.md index 49796a2..869601a 100644 --- a/influxdata/basic_transformation/README.md +++ b/influxdata/basic_transformation/README.md @@ -56,8 +56,7 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor - [basic_transformation_config_scheduler.toml](basic_transformation_config_scheduler.toml) - for scheduled triggers - [basic_transformation_config_data_writes.toml](basic_transformation_config_data_writes.toml) - for data write triggers -For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins -/README.md](/README.md). +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](../../README.md). ## Data requirements @@ -69,7 +68,7 @@ The plugin assumes that the table schema is already defined in the database, as - **Python packages**: - `pint` (for unit conversions) -### Installation steps +## Installation steps 1. Start InfluxDB 3 with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): diff --git a/scripts/validate_readme.py b/scripts/validate_readme.py index 4acbe84..7879c7b 100644 --- a/scripts/validate_readme.py +++ b/scripts/validate_readme.py @@ -17,7 +17,7 @@ SUMMARY_SEPARATOR_LENGTH = 60 # Section search offsets -SECTION_SEARCH_OFFSET = 1 +SECTION_SEARCH_OFFSET = 100 SECTION_NOT_FOUND = -1 # Exit codes @@ -95,7 +95,7 @@ def validate_parameter_tables(content: str) -> List[str]: errors = [] # Check for parameter table headers - if '| Parameter | Type | Default | Description |' not in content: + if not re.search(r'\|\s*Parameter\s*\|\s*Type\s*\|\s*Default\s*\|\s*Description\s*\|', content): errors.append("No properly formatted parameter tables found (should have Parameter | Type | Default | Description columns)") # Check for required parameters section From ff44a0875d8df4302f26569dec0929b8423073a4 Mon Sep 17 00:00:00 2001 From: meelahme Date: Fri, 10 Oct 2025 17:28:55 -0700 Subject: [PATCH 5/7] fix: eliminate validation false positives with robust section extraction --- scripts/validate_readme.py | 182 +++++++++++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 40 deletions(-) diff --git a/scripts/validate_readme.py b/scripts/validate_readme.py index 7879c7b..64a3b71 100644 --- a/scripts/validate_readme.py +++ b/scripts/validate_readme.py @@ -50,6 +50,67 @@ "### Performance considerations" ] +def extract_section_content(content: str, section_heading: str) -> str: + """ + Extract content between a markdown section heading and the next same-level heading. + + This function properly handles: + - Subsections (###, ####) within the section + - Code blocks containing ## characters + - Section being at start or end of document + - Missing sections + + Args: + content: Full markdown document content + section_heading: Heading text without the ## prefix (e.g., "Troubleshooting") + + Returns: + Section content as string, or empty string if section not found + + Examples: + >>> extract_section_content(doc, "Troubleshooting") + '### Common issues\\n#### Issue: ...' + """ + # Build the section marker - look for "## Heading" where Heading doesn't start with # + # This ensures we match "## Troubleshooting" but not "### Troubleshooting" + section_pattern = f'## {section_heading}' + + # Find the section heading + section_index = content.find(section_pattern) + if section_index == -1: + return "" + + # Find the end of the heading line (start of content) + content_start = content.find('\n', section_index) + if content_start == -1: + # Section heading is at end of file with no content + return "" + content_start += 1 # Move past the newline + + # Find the next same-level heading (## followed by space, not ###) + # Use a loop to ensure we're not matching subsections + search_pos = content_start + while True: + next_heading_pos = content.find('\n## ', search_pos) + + if next_heading_pos == -1: + # No more headings, take everything to end of document + return content[content_start:].rstrip() + + # Check that it's actually a ## heading and not ### + # Look at the character after '## ' + check_pos = next_heading_pos + 4 # Position after '\n## ' + if check_pos < len(content) and content[check_pos] != '#': + # This is a proper ## heading, not ### or #### + return content[content_start:next_heading_pos].rstrip() + + # This was a false match (like \n### ), keep searching + search_pos = next_heading_pos + 1 + + # Safety: if we've searched too far, something is wrong + if search_pos > len(content): + return content[content_start:].rstrip() + def validate_emoji_metadata(content: str) -> List[str]: """Validate the emoji metadata line.""" errors = [] @@ -94,20 +155,38 @@ def validate_parameter_tables(content: str) -> List[str]: """Validate parameter table formatting.""" errors = [] - # Check for parameter table headers - if not re.search(r'\|\s*Parameter\s*\|\s*Type\s*\|\s*Default\s*\|\s*Description\s*\|', content): + # Check for parameter table headers with flexible whitespace + # This regex allows variable spacing between columns + table_pattern = r'\|\s*Parameter\s*\|\s*Type\s*\|\s*Default\s*\|\s*Description\s*\|' + if not re.search(table_pattern, content): errors.append("No properly formatted parameter tables found (should have Parameter | Type | Default | Description columns)") - # Check for required parameters section + # Validate Required parameters section if present if '### Required parameters' in content: - section_start = content.index('### Required parameters') - section_end = content.find('\n###', section_start + SECTION_SEARCH_OFFSET) - if section_end == SECTION_NOT_FOUND: - section_end = content.find('\n##', section_start + SECTION_SEARCH_OFFSET) + # Find the subsection + subsection_start = content.find('### Required parameters') + + # Find the end: next subsection (###) or next section (##) + # Look for '\n### ' or '\n## ' after current position + search_pos = subsection_start + len('### Required parameters') + + next_subsection = content.find('\n### ', search_pos) + next_section = content.find('\n## ', search_pos) - section_content = content[section_start:section_end] if section_end != SECTION_NOT_FOUND else content[section_start:] + # Determine the end position + if next_subsection == -1 and next_section == -1: + subsection_end = len(content) + elif next_subsection == -1: + subsection_end = next_section + elif next_section == -1: + subsection_end = next_subsection + else: + subsection_end = min(next_subsection, next_section) + + section_content = content[subsection_start:subsection_end] - if 'required' not in section_content.lower(): + # Check that it indicates which parameters are required + if 'required' not in section_content.lower() and 'yes' not in section_content.lower(): errors.append("Required parameters section should indicate which parameters are required") return errors @@ -166,23 +245,32 @@ def validate_troubleshooting(content: str) -> List[str]: """Validate troubleshooting section content.""" errors = [] - if '## Troubleshooting' in content: - section_start = content.index('## Troubleshooting') - section_end = content.find('\n##', section_start + SECTION_SEARCH_OFFSET) - section_content = content[section_start:section_end] if section_end != SECTION_NOT_FOUND else content[section_start:] - - # Check for common subsections - if '### Common issues' not in section_content: - errors.append("Troubleshooting should include 'Common issues' subsection") - - # Check for issue/solution pattern - issue_count = section_content.count('#### Issue:') + section_content.count('**Issue:') - solution_count = section_content.count('**Solution:') + section_content.count('Solution:') - - if issue_count < MINIMUM_TROUBLESHOOTING_ISSUES: - errors.append(f"Troubleshooting should include at least {MINIMUM_TROUBLESHOOTING_ISSUES} documented issues") - if issue_count > solution_count: - errors.append("Each troubleshooting issue should have a corresponding solution") + # Check if section exists + if '## Troubleshooting' not in content: + return errors + + # Extract the section content using robust extraction + section_content = extract_section_content(content, 'Troubleshooting') + + # Defensive check + if not section_content.strip(): + errors.append("Troubleshooting section exists but appears empty") + return errors + + # Check for required subsections + if '### Common issues' not in section_content: + errors.append("Troubleshooting should include 'Common issues' subsection") + + # Check for issue/solution patterns + # Count both markdown heading style (####) and bold style (**) + issue_count = section_content.count('#### Issue:') + section_content.count('**Issue:') + solution_count = (section_content.count('**Solution**:') + section_content.count('**Solution:') + section_content.count('Solution:')) + + if issue_count < MINIMUM_TROUBLESHOOTING_ISSUES: + errors.append(f"Troubleshooting should include at least {MINIMUM_TROUBLESHOOTING_ISSUES} documented issues") + + if issue_count > 0 and solution_count < issue_count: + errors.append("Each troubleshooting issue should have a corresponding solution") return errors @@ -190,20 +278,34 @@ def validate_code_overview(content: str) -> List[str]: """Validate code overview section.""" errors = [] - if '## Code overview' in content: - section_start = content.index('## Code overview') - section_end = content.find('\n##', section_start + SECTION_SEARCH_OFFSET) - section_content = content[section_start:section_end] if section_end != SECTION_NOT_FOUND else content[section_start:] - - # Check for required subsections - if '### Files' not in section_content: - errors.append("Code overview should include 'Files' subsection") - if '### Main functions' not in section_content and '### Key functions' not in section_content: - errors.append("Code overview should include 'Main functions' or 'Key functions' subsection") - - # Check for function documentation - if 'def ' not in section_content and not re.search(r'`\w+\(.*?\)`', section_content): - errors.append("Code overview should document main functions with their signatures") + # Check if section exists + if '## Code overview' not in content: + return errors + + # Extract the section content using robust extraction + section_content = extract_section_content(content, 'Code overview') + + # Defensive check + if not section_content.strip(): + errors.append("Code overview section exists but appears empty") + return errors + + # Check for required subsections + if '### Files' not in section_content: + errors.append("Code overview should include 'Files' subsection") + + if not ('### Main functions' in section_content or '### Key functions' in section_content): + errors.append("Code overview should include 'Main functions' or 'Key functions' subsection") + + # Check for function documentation (signatures in backticks or Python def) + has_function_signatures = ( + 'def ' in section_content or + re.search(r'####?\s+`\w+\(.*?\)`', section_content) or + re.search(r'`\w+\([^)]*\)`:', section_content) + ) + + if not has_function_signatures: + errors.append("Code overview should document main functions with their signatures") return errors From de990aef89349cfa808a4cce2b8105ec46f8aa0f Mon Sep 17 00:00:00 2001 From: meelahme Date: Fri, 10 Oct 2025 17:39:28 -0700 Subject: [PATCH 6/7] fix: remove arbitrary 10-file limit from README change detection --- .github/workflows/remind-sync-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/remind-sync-docs.yml b/.github/workflows/remind-sync-docs.yml index 4de28fd..85a437d 100644 --- a/.github/workflows/remind-sync-docs.yml +++ b/.github/workflows/remind-sync-docs.yml @@ -23,7 +23,7 @@ jobs: id: detect-changes run: | # Get list of changed README files - CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep '^influxdata/.*/README\.md$' | head -10) + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep '^influxdata/.*/README\.md$') if [[ -z "$CHANGED_FILES" ]]; then echo "No plugin README files changed" From 3c2bc222080a668806ae4378faeecefca9f5c30f Mon Sep 17 00:00:00 2001 From: meelahme Date: Fri, 10 Oct 2025 17:54:04 -0700 Subject: [PATCH 7/7] refactor: address code quality feedback from Copilot review --- scripts/validate_readme.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/validate_readme.py b/scripts/validate_readme.py index 64a3b71..ff6a731 100644 --- a/scripts/validate_readme.py +++ b/scripts/validate_readme.py @@ -23,6 +23,7 @@ # Exit codes EXIT_SUCCESS = 0 EXIT_ERROR = 1 +EMOJI_METADATA_PATTERN = r'^āš”\s+[\w\-,\s]+\s+šŸ·ļø\s+[\w\-,\s]+\s+šŸ”§\s+InfluxDB 3' # Required sections in order REQUIRED_SECTIONS = [ @@ -50,6 +51,8 @@ "### Performance considerations" ] +OPTIONAL_SECTIONS_SKIP_WARNING = ["### Debugging tips", "### Performance considerations"] + def extract_section_content(content: str, section_heading: str) -> str: """ Extract content between a markdown section heading and the next same-level heading. @@ -116,8 +119,7 @@ def validate_emoji_metadata(content: str) -> List[str]: errors = [] # Check for emoji metadata pattern - metadata_pattern = r'^āš”\s+[\w\-,\s]+\s+šŸ·ļø\s+[\w\-,\s]+\s+šŸ”§\s+InfluxDB 3' - if not re.search(metadata_pattern, content, re.MULTILINE): + if not re.search(EMOJI_METADATA_PATTERN, content, re.MULTILINE): errors.append("Missing or invalid emoji metadata line (should have āš” trigger types šŸ·ļø tags šŸ”§ compatibility)") return errors @@ -355,7 +357,7 @@ def validate_readme(readme_path: Path) -> Tuple[List[str], List[str]]: # Check for optional but recommended sections for section in OPTIONAL_SECTIONS: - if section not in content and section not in ["### Debugging tips", "### Performance considerations"]: + if section not in content and section not in OPTIONAL_SECTIONS_SKIP_WARNING: warnings.append(f"Consider adding '{section}' section") # Check for template remnants