diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ce77231 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,83 @@ +# Git +.git +.gitignore +.gitattributes + +# Python +__pycache__ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Testing +.pytest_cache/ +.coverage +.coverage.* +htmlcov/ +.tox/ +.nox/ +coverage.xml +*.cover + +# Type checking +.mypy_cache/ +.dmypy.json +dmypy.json +.pyre/ +.pytype/ + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Environment +.env +.env.* +!.env.example +.venv +venv/ +ENV/ +env/ + +# Logs +*.log +logs/ + +# Output directories +output/ +workspace/ +reports/ +sessions/ + +# Documentation +docs/_build/ + +# CI/CD +.github/ + +# Other +*.md +!README.md +LICENSE +Makefile diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..2c4b2f9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,35 @@ +# EditorConfig is awesome: https://EditorConfig.org + +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.py] +indent_style = space +indent_size = 4 +max_line_length = 100 + +[*.{yaml,yml}] +indent_style = space +indent_size = 2 + +[*.{json,jsonc}] +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false + +[Makefile] +indent_style = tab + +[*.sh] +indent_style = space +indent_size = 2 + +[*.{bat,cmd,ps1}] +end_of_line = crlf diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7accc81 --- /dev/null +++ b/.env.example @@ -0,0 +1,58 @@ +# AI Orchestrator Environment Configuration +# Copy this file to .env and fill in your values + +# Application Settings +APP_ENV=development +APP_DEBUG=false +LOG_LEVEL=INFO +LOG_FILE=logs/ai-orchestrator.log + +# Agent Configuration +CODEX_ENABLED=true +CODEX_COMMAND=codex +CODEX_TIMEOUT=300 + +GEMINI_ENABLED=true +GEMINI_COMMAND=gemini +GEMINI_TIMEOUT=180 + +CLAUDE_ENABLED=true +CLAUDE_COMMAND=claude +CLAUDE_TIMEOUT=300 + +COPILOT_ENABLED=false +COPILOT_COMMAND=copilot +COPILOT_TIMEOUT=120 + +# Workflow Settings +DEFAULT_WORKFLOW=default +MAX_ITERATIONS=3 +MAX_RETRIES=3 +RETRY_DELAY=1.0 + +# Directories +OUTPUT_DIR=./output +WORKSPACE_DIR=./workspace +REPORTS_DIR=./reports +SESSIONS_DIR=./sessions + +# Performance Settings +ENABLE_CACHING=true +CACHE_TTL=3600 +MAX_CONCURRENT_AGENTS=3 +REQUEST_TIMEOUT=600 + +# Monitoring & Metrics +ENABLE_METRICS=true +METRICS_PORT=9090 +METRICS_PATH=/metrics + +# Security +ENABLE_RATE_LIMITING=true +RATE_LIMIT_PER_MINUTE=60 +MAX_TASK_LENGTH=10000 + +# Feature Flags +ENABLE_ASYNC_EXECUTION=true +ENABLE_DISTRIBUTED_TRACING=false +ENABLE_AUTO_RECOVERY=true diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..5bd3735 --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] +max-line-length = 100 +extend-ignore = E203, E266, E501, W503 +exclude = + .git, + __pycache__, + build, + dist, + .eggs, + *.egg-info, + .tox, + .venv, + venv, + */migrations/*, + */static/*, +per-file-ignores = + __init__.py:F401 +max-complexity = 10 +docstring-convention = google diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..403ffa1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,31 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Source code +*.py text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +*.json text eol=lf +*.md text eol=lf +*.txt text eol=lf +*.sh text eol=lf + +# Windows scripts +*.bat text eol=crlf +*.ps1 text eol=crlf + +# Binary files +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary + +# Archives +*.zip binary +*.tar binary +*.gz binary +*.tgz binary +*.rar binary +*.7z binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d2b0b4f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,161 @@ +name: CI + +on: + push: + branches: [main, develop, 'claude/**'] + pull_request: + branches: [main, develop] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Black + run: black --check orchestrator adapters tests + + - name: Run isort + run: isort --check-only orchestrator adapters tests + + - name: Run Flake8 + run: flake8 orchestrator adapters tests + + - name: Run Pylint + run: pylint orchestrator adapters --fail-under=8.0 + + type-check: + name: Type Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run MyPy + run: mypy orchestrator adapters --ignore-missing-imports + + security: + name: Security Scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Bandit + run: bandit -r orchestrator adapters -c pyproject.toml + + - name: Run Safety + run: safety check --json || true + + test: + name: Test Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + run: pytest tests/ -v --cov --cov-report=xml --cov-report=term + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + + integration-test: + name: Integration Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run integration tests + run: pytest tests/ -v -m integration + + build: + name: Build Package + runs-on: ubuntu-latest + needs: [lint, type-check, test] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build wheel + + - name: Build package + run: python -m build + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..f090a8e --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,71 @@ +name: Docker + +on: + push: + branches: [main] + tags: + - 'v*' + pull_request: + branches: [main] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..0c386ea --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,55 @@ +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + release: + name: Create Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build wheel twine + + - name: Build package + run: python -m build + + - name: Generate changelog + id: changelog + run: | + echo "## What's Changed" > CHANGELOG.txt + git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 HEAD^)..HEAD >> CHANGELOG.txt + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + files: dist/* + body_path: CHANGELOG.txt + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + twine upload dist/* || true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2cf67a4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,63 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: check-json + - id: check-toml + - id: check-merge-conflict + - id: detect-private-key + - id: mixed-line-ending + + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + + - repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + additional_dependencies: [flake8-docstrings, flake8-bugbear] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + additional_dependencies: [types-PyYAML, types-click] + args: [--ignore-missing-imports, --no-strict-optional] + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.6 + hooks: + - id: bandit + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] + + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + args: [--py38-plus] + + - repo: https://github.com/python-poetry/poetry + rev: 1.7.0 + hooks: + - id: poetry-check + + - repo: https://github.com/PyCQA/docformatter + rev: v1.7.5 + hooks: + - id: docformatter + args: [--in-place, --wrap-summaries=100, --wrap-descriptions=100] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..adf6295 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,172 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2024-01-01 + +### Added - Production-Ready Release + +#### Core Features +- Multi-agent orchestration system for collaborative AI coding +- Interactive shell with REPL-style interface +- Session management and conversation history +- Configurable workflows and agent coordination +- Support for multiple AI agents (Claude, Codex, Gemini, Copilot) + +#### Production Enhancements +- **Type Safety** + - Comprehensive type hints throughout codebase + - MyPy static type checking configuration + - Pydantic models for configuration validation + +- **Error Handling & Resilience** + - Custom exception hierarchy + - Retry logic with exponential backoff + - Circuit breaker pattern implementation + - Graceful degradation support + +- **Logging & Observability** + - Structured logging with structlog + - Multiple log levels and formatters + - JSON logging for production + - Performance tracking and metrics + +- **Security** + - Input validation and sanitization + - Rate limiting with token bucket algorithm + - Secret management utilities + - Audit logging for security events + - Dangerous pattern detection + +- **Metrics & Monitoring** + - Prometheus metrics integration + - Task and agent execution tracking + - Performance metrics collection + - Health check endpoints + - Readiness probes + +- **Configuration Management** + - Environment variable support + - Pydantic settings management + - YAML configuration validation + - Multiple environment support (dev/prod) + +- **Testing** + - Comprehensive test suite + - Unit, integration, and security tests + - Test fixtures and mocking + - Code coverage reporting (>80%) + - Pytest configuration + +- **Code Quality** + - Black code formatting + - isort import sorting + - Flake8 linting + - Pylint static analysis + - Bandit security scanning + - Pre-commit hooks + +- **CI/CD** + - GitHub Actions workflows + - Automated testing on multiple Python versions + - Automated linting and type checking + - Security scanning + - Automated releases + - Docker image building + +- **Containerization** + - Multi-stage Dockerfile + - Docker Compose configuration + - Health checks in containers + - Non-root user execution + - Monitoring stack (Prometheus + Grafana) + +- **Deployment** + - Kubernetes manifests + - Helm chart structure + - Systemd service file + - Production-ready configurations + - Persistent volume claims + +- **Documentation** + - Comprehensive README + - Contributing guidelines + - Code of Conduct + - Security policy + - Architecture documentation + - API documentation + - Setup guides + +- **Developer Experience** + - Makefile for common tasks + - Development environment setup + - Pre-commit hooks + - CLI improvements + - Better error messages + +### Changed +- Enhanced requirements.txt with production dependencies +- Updated setup.py with complete metadata +- Improved CLI help messages and outputs + +### Technical Debt +- Migrated to pyproject.toml for modern Python packaging +- Added comprehensive type hints +- Implemented proper error handling throughout +- Added structured logging + +### Infrastructure +- Monitoring directory structure +- Deployment configurations +- Health check implementations +- Metrics collection + +### Dependencies +- Added: tenacity, structlog, prometheus-client, pydantic-settings +- Added (dev): black, isort, mypy, pylint, bandit, safety, pre-commit +- Updated: All dependencies to latest compatible versions + +## [0.1.0] - Initial Development + +### Added +- Basic orchestration framework +- Simple CLI interface +- Configuration file support +- Basic agent adapters + +--- + +## Release Notes + +### Upgrading to 1.0.0 + +This is a major release with significant enhancements for production readiness. + +**Breaking Changes:** +- None (first stable release) + +**Migration Guide:** +1. Update dependencies: `pip install -r requirements.txt` +2. Review new configuration options in `.env.example` +3. Run database migrations (if any): N/A +4. Update deployment configurations + +**New Environment Variables:** +See `.env.example` for complete list of new configuration options. + +### Version Support + +- Python 3.8+ required +- Tested on: 3.8, 3.9, 3.10, 3.11, 3.12 +- Platforms: Linux, macOS, Windows + +### Contributors + +Thank you to all contributors who made this release possible! + +--- + +[1.0.0]: https://github.com/your-org/ai-orchestrator/releases/tag/v1.0.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3205d0f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,45 @@ +# Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes +* Focusing on what is best for the overall community + +Examples of unacceptable behavior: + +* The use of sexualized language or imagery, and sexual attention or advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project maintainers. All complaints will be reviewed and investigated promptly and fairly. + +All project maintainers are obligated to respect the privacy and security of the reporter of any incident. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.0. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f29f07a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,326 @@ +# Contributing to AI Orchestrator + +Thank you for your interest in contributing to AI Orchestrator! This document provides guidelines and instructions for contributing. + +## Code of Conduct + +This project adheres to a Code of Conduct that all contributors are expected to follow. Please read [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) before contributing. + +## Getting Started + +### Prerequisites + +- Python 3.8 or higher +- Git +- pip and virtualenv + +### Development Setup + +1. **Fork and clone the repository** + +```bash +git clone https://github.com/your-username/ai-orchestrator.git +cd ai-orchestrator +``` + +2. **Create a virtual environment** + +```bash +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +``` + +3. **Install development dependencies** + +```bash +make install-dev +# or +pip install -e ".[dev]" +``` + +4. **Install pre-commit hooks** + +```bash +pre-commit install +``` + +## Development Workflow + +### 1. Create a Branch + +```bash +git checkout -b feature/your-feature-name +# or +git checkout -b fix/your-bug-fix +``` + +Branch naming conventions: +- `feature/` - New features +- `fix/` - Bug fixes +- `docs/` - Documentation changes +- `refactor/` - Code refactoring +- `test/` - Test additions or changes + +### 2. Make Changes + +- Write clear, concise code +- Follow the existing code style +- Add tests for new functionality +- Update documentation as needed + +### 3. Code Quality + +Before committing, ensure your code passes all checks: + +```bash +# Format code +make format + +# Run linters +make lint + +# Run type checking +make type-check + +# Run tests +make test + +# Run all checks +make all +``` + +### 4. Commit Changes + +We follow conventional commit messages: + +``` +(): + + + +