diff --git a/.gitignore b/.gitignore index 5e1325f5a2c..8a7855f9fec 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,9 @@ releases/*.tar.gz # Conversation history (contains potential secrets) *.txt *.py +!scripts/fast_build.py +!scripts/resolve_merge_conflicts.py +!scripts/upstream_sync.py C:\Users\downl\Desktop\codex-main\.specstory\history\2026-01-16_05-43Z-未来のai開発と統合エコシステム.md codex-rs/build_error.txt codex-rs/build_error.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 372e726bd1a..cb13e69d686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,19 +128,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### ✅ Added -- **Fast Incremental Build System (`scripts/fast_build.py`)** +- **Fast Incremental Build System (`scripts/fast_build.py` + `scripts/upstream_sync.py`)** - MD5 hash-based change detection for intelligent rebuilds - Cargo incremental compilation optimization - Parallel build processing with CPU core utilization - tqdm-powered progress visualization - - Build cache persistence (`.build_cache.pkl`) + - Build cache persistence (`.codex-fast-build-cache.json`) -- **Hot Reload Installation System (`scripts/build_and_install.py`)** +- **Hot Reload Installation System (`scripts/fast_build.py fast-build-install`)** - Cross-platform process detection and termination (psutil) - Atomic binary replacement with safety checks - Platform-specific installation (Windows/macOS/Linux) - Installation verification with version checking - - PowerShell integration for Windows deployment + - PowerShell wrapper (`codex-rs/fast_build.ps1`) plus `just fast-build*` entry points - **Integrated Release Packaging** - GitHub Actions workflow for cross-platform tgz packages @@ -151,8 +151,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Development Tools Enhancement** - `just fast-build` - Quick incremental builds - - `just build-install` - Full pipeline execution - - `just install-kill` - Direct binary replacement + - `just fast-build-install` - Full pipeline execution + - `just upstream-sync` - Upstream merge + resolver orchestration - Process-safe deployment with zero-downtime updates ### 🎯 Performance Improvements diff --git a/CLAUDE.md b/CLAUDE.md index 8c520cd99a5..08aeaa25157 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,9 @@ just clippy # Lint all crates just fix -p # Auto-fix clippy issues for a specific crate just test # Run all tests with cargo-nextest just install # Fetch dependencies, show active toolchain +just fast-build [targets] # Differential build via scripts/fast_build.py +just fast-build-install # Differential build + install via scripts/fast_build.py +just upstream-sync # Fetch/merge upstream and run resolver just codex # Run codex from source (cargo run --bin codex) just exec "prompt" # Run codex exec mode @@ -40,11 +43,12 @@ cargo shear # Find unused dependencies cargo deny check # License/security audit ``` -### Windows fast-build scripts (from codex-rs/) +### Cross-platform fast-build tasks ```powershell -.\ultra-fast-build-install.ps1 # Fastest incremental build -.\clean-build-install.ps1 # Clean rebuild +just fast-build codex-cli codex-tui # Differential build +just fast-build-install codex-cli codex-tui # Differential build + install +.\fast_build.ps1 -Task fast-build-install # Windows wrapper around scripts/fast_build.py ``` ### Bazel (alternative build system) diff --git a/codex-rs/README.md b/codex-rs/README.md index 1cddcd9e42f..34b47164ff9 100644 --- a/codex-rs/README.md +++ b/codex-rs/README.md @@ -62,12 +62,12 @@ Codex-RSは、依存関係の重い環境から脱却し、**単一のネイテ **Built for speed.** - **sccache Support**: Optimized for 6-core parallel builds, drastically reducing compilation time. -- **Atomic Updates**: Smart installation scripts (`fast_build.ps1`) ensuring zero-downtime updates during development. +- **Atomic Updates**: `just fast-build-install` and `fast_build.ps1` wrap a shared Python pipeline for zero-downtime updates during development. **スピードを追求** - **sccacheサポート**: 6コア並列ビルドに最適化されており、コンパイル時間を大幅に短縮。 -- **アトミック更新**: スマートなインストールスクリプト(`fast_build.ps1`)により、開発中のゼロダウンタイム更新を実現。 +- **アトミック更新**: `just fast-build-install` と `fast_build.ps1` が共通Pythonパイプラインを呼び出し、開発中のゼロダウンタイム更新を実現。 --- @@ -100,9 +100,11 @@ For developers who value speed, utilize our optimized PowerShell workflow: スピードを重視する開発者向けに、最適化されたPowerShellワークフローを提供しています: ```powershell -# Optimized for 6-core parallel processing -# 6コア並列処理に最適化 -.\fast_build.ps1 +# Default: 6 parallel jobs (override with CODEX_FAST_BUILD_JOBS or --jobs) +# 既定: 6並列(`CODEX_FAST_BUILD_JOBS` または `--jobs` で上書き) +just fast-build-install codex-cli codex-tui +# Windows wrapper +.\fast_build.ps1 -Task fast-build-install -Targets codex-cli,codex-tui ``` ### Standard Usage / 基本的な使い方 diff --git a/codex-rs/fast_build.ps1 b/codex-rs/fast_build.ps1 index 85ae5124d67..98b115842de 100644 --- a/codex-rs/fast_build.ps1 +++ b/codex-rs/fast_build.ps1 @@ -1,62 +1,46 @@ -# Fast Build & Install Script (Simplified) -# Requirements: 6-core, sccache, kill processes, overwrite install - -$ErrorActionPreference = "Stop" - -function Write-Status($msg) { Write-Host "[$(Get-Date -Format 'HH:mm:ss')] $msg" -ForegroundColor Cyan } -function Write-Success($msg) { Write-Host "[$(Get-Date -Format 'HH:mm:ss')] $msg" -ForegroundColor Green } - -# 1. Setup Environment -if ($env:USE_SCCACHE -eq "1" -and (Get-Command sccache -ErrorAction SilentlyContinue)) { - $env:RUSTC_WRAPPER = "sccache" - $cacheMode = "sccache" -} else { - Remove-Item Env:RUSTC_WRAPPER -ErrorAction SilentlyContinue - $env:SCCACHE_DISABLE = "1" - $cacheMode = "rustc" +[CmdletBinding()] +param( + [ValidateSet('fast-build', 'fast-build-install')] + [string]$Task = 'fast-build-install', + [ValidateSet('md5', 'mtime', 'cargo-metadata')] + [string]$Method = $(if ($env:CODEX_FAST_BUILD_METHOD) { $env:CODEX_FAST_BUILD_METHOD } else { 'md5' }), + [int]$Jobs = $(if ($env:CODEX_FAST_BUILD_JOBS) { [int]$env:CODEX_FAST_BUILD_JOBS } else { 6 }), + [string[]]$Targets = @('codex-cli', 'codex-tui', 'codex-gui'), + [string]$LogFile, + [switch]$Force, + [switch]$NoDenyWarnings +) + +$ErrorActionPreference = 'Stop' +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$scriptPath = Join-Path $repoRoot 'scripts\fast_build.py' +$python = if (Get-Command py -ErrorAction SilentlyContinue) { 'py' } elseif (Get-Command python -ErrorAction SilentlyContinue) { 'python' } else { throw 'Python launcher not found (expected py or python).' } + +$argsList = @() +if ($python -eq 'py') { + $argsList += '-3' } -$env:RUSTFLAGS = "-D warnings" -Write-Status "Environment configured: compiler=$cacheMode, Jobs=6" - -# 2. Kill Processes -Write-Status "Killing existing processes..." -Stop-Process -Name "codex" -Force -ErrorAction SilentlyContinue -Stop-Process -Name "codex-tui" -Force -ErrorAction SilentlyContinue -Stop-Process -Name "codex-gui" -Force -ErrorAction SilentlyContinue -Start-Sleep -Seconds 1 - -# 3. Build (Direct execution for visibility) -Write-Status "Starting Cargo Build (CLI + TUI)..." -# We build both. If CLI is already built, sccache/cargo will skip it quickly. -cargo build --release -p codex-cli -p codex-tui -j 6 - -if ($LASTEXITCODE -ne 0) { - Write-Error "Build failed with code $LASTEXITCODE" - exit $LASTEXITCODE +$argsList += $scriptPath +$argsList += $Task +$argsList += '--changed-only' +$argsList += '--jobs' +$argsList += $Jobs +$argsList += '--method' +$argsList += $Method +if ($LogFile) { + $argsList += '--log-file' + $argsList += $LogFile } - -Write-Success "Build Complete." - -# 4. Install (Overwrite) -Write-Status "Installing binaries..." -$installDir = "$env:USERPROFILE\.cargo\bin" -if (-not (Test-Path $installDir)) { New-Item -ItemType Directory -Path $installDir -Force } - -$binaries = @("codex.exe", "codex-tui.exe") -foreach ($bin in $binaries) { - $src = "target\release\$bin" - $dest = "$installDir\$bin" - - if (Test-Path $src) { - Write-Host "Copying $bin to $dest..." - Copy-Item -Path $src -Destination $dest -Force - Write-Success "Installed $bin" - } - else { - Write-Warning "$bin not found in target/release!" - } +if ($Force) { + $argsList += '--force' +} +if ($NoDenyWarnings) { + $argsList += '--no-deny-warnings' +} +if ($Targets.Count -gt 0) { + $argsList += $Targets } -# 5. Verify -Write-Status "Verifying installation..." -& "$installDir\codex.exe" --version +Write-Host "[fast_build.ps1] $python $($argsList -join ' ')" -ForegroundColor Cyan +& $python @argsList +exit $LASTEXITCODE diff --git a/codex-rs/justfile b/codex-rs/justfile index f37dd3eb8a3..611f1bdd9e9 100644 --- a/codex-rs/justfile +++ b/codex-rs/justfile @@ -1,5 +1,7 @@ set positional-arguments +python_cmd := if os_family() == "windows" { "py -3" } else { "python3" } +build_script := "../scripts/fast_build.py" # Display help help: just -l @@ -58,4 +60,13 @@ install-tui: build-install-all: cargo build --release -p codex-cli -p codex-tui cargo install --path cli --force - cargo install --path tui --force \ No newline at end of file + cargo install --path tui --force + +fast-build *args: + {{python_cmd}} {{build_script}} fast-build --changed-only "$@" + +fast-build-install *args: + {{python_cmd}} {{build_script}} fast-build-install --changed-only "$@" + +upstream-sync *args: + {{python_cmd}} {{build_script}} upstream-sync "$@" diff --git "a/docs/development/2026-01-04_\351\253\230\351\200\237\343\203\223\343\203\253\343\203\211\343\203\220\343\202\244\343\203\212\343\203\252\343\202\244\343\203\263\343\202\271\343\203\210\343\203\274\343\203\253\345\256\237\346\251\237\343\203\206\343\202\271\343\203\210\345\256\214\344\272\206.md" "b/docs/development/2026-01-04_\351\253\230\351\200\237\343\203\223\343\203\253\343\203\211\343\203\220\343\202\244\343\203\212\343\203\252\343\202\244\343\203\263\343\202\271\343\203\210\343\203\274\343\203\253\345\256\237\346\251\237\343\203\206\343\202\271\343\203\210\345\256\214\344\272\206.md" index 224d7624308..d1c7ae8ff80 100644 --- "a/docs/development/2026-01-04_\351\253\230\351\200\237\343\203\223\343\203\253\343\203\211\343\203\220\343\202\244\343\203\212\343\203\252\343\202\244\343\203\263\343\202\271\343\203\210\343\203\274\343\203\253\345\256\237\346\251\237\343\203\206\343\202\271\343\203\210\345\256\214\344\272\206.md" +++ "b/docs/development/2026-01-04_\351\253\230\351\200\237\343\203\223\343\203\253\343\203\211\343\203\220\343\202\244\343\203\212\343\203\252\343\202\244\343\203\263\343\202\271\343\203\210\343\203\274\343\203\253\345\256\237\346\251\237\343\203\206\343\202\271\343\203\210\345\256\214\344\272\206.md" @@ -8,7 +8,7 @@ - Cargo.lockの再生成 ### 2. 高速差分ビルドの実行 -- `scripts/fast_build.py debug codex-cli` を実行 +- `python3 scripts/fast_build.py fast-build --changed-only codex-cli --profile debug` を実行 - ファイル変更検出(1363ファイル)完了 - フルビルドが必要と判定 @@ -26,7 +26,7 @@ - タイトル: "Codex Extended - Skills + MCP + Agents SDK Architecture" - アーキテクチャ図: Mermaidで生成したv2.10.0アーキテクチャ図 - 機能説明: Skills System, MCP Integration, Agents SDK Patterns -- 使用例: `codex $ build-manager fast-build` などの新コマンド +- 使用例: `just fast-build`, `just fast-build-install`, `just upstream-sync` などの正式タスク - 機能ステータスマトリックス更新 - ドキュメントリンク更新 diff --git a/docs/plan/MERGE_STRATEGY.md b/docs/plan/MERGE_STRATEGY.md index cc0eee4d1ab..8c1df456b1d 100644 --- a/docs/plan/MERGE_STRATEGY.md +++ b/docs/plan/MERGE_STRATEGY.md @@ -18,8 +18,8 @@ This document describes the strategy for merging upstream OpenAI Codex changes w - `zapabob/scripts/load-env.sh` - Environment loading script - `zapabob/scripts/setup-env-vars.ps1` - PowerShell environment setup -- `merge_with_custom_features.py` - Custom merge tool -- `advanced_merge_resolver.py` - Advanced merge conflict resolver +- `scripts/upstream_sync.py` - Upstream merge orchestration entry point +- `scripts/resolve_merge_conflicts.py` - Upstream-first merge conflict resolver ## Merge Strategy @@ -28,7 +28,7 @@ This document describes the strategy for merging upstream OpenAI Codex changes w 1. **Identify Custom Features** ```bash - python3 advanced_merge_resolver.py --identify + python3 scripts/resolve_merge_conflicts.py codex-rs/tui/src/slash_command.rs --rule "codex-rs/tui/src/slash_command.rs=upstream-reinject" ``` 2. **Backup Current State** @@ -58,7 +58,7 @@ git diff --name-only --diff-filter=U #### Automatic Merge ```bash -python3 advanced_merge_resolver.py +python3 scripts/upstream_sync.py ``` #### Manual Merge (if needed) @@ -77,7 +77,8 @@ python3 advanced_merge_resolver.py ```bash # Check custom features preserved -python3 advanced_merge_resolver.py --verify +git diff --check +python3 scripts/fast_build.py fast-build --changed-only codex-cli codex-tui # Run tests cd codex-rs && cargo test -p codex-tui @@ -115,7 +116,7 @@ Custom cmd Modified → Merge, preserve custom **Solution**: Run the merge resolver ```bash -python3 advanced_merge_resolver.py --file codex-rs/tui/src/slash_command.rs +python3 scripts/resolve_merge_conflicts.py codex-rs/tui/src/slash_command.rs --rule "codex-rs/tui/src/slash_command.rs=upstream-reinject" ``` #### 2. Custom Features Not Detected @@ -124,10 +125,9 @@ python3 advanced_merge_resolver.py --file codex-rs/tui/src/slash_command.rs ```bash python3 -c " -from advanced_merge_resolver import MergeConflictResolver -resolver = MergeConflictResolver() -features = resolver.identify_custom_features() -print(features) +from scripts.resolve_merge_conflicts import choose_strategy, load_rules +rules = load_rules([]) +print(choose_strategy("codex-rs/tui/src/slash_command.rs", rules)) " ``` @@ -163,12 +163,13 @@ if git merge upstream/main --no-edit; then echo "Merge successful!" else echo "Conflicts detected, running resolver..." - python3 advanced_merge_resolver.py + python3 scripts/upstream_sync.py fi # Step 4: Verify echo "[4/4] Verifying custom features..." -python3 advanced_merge_resolver.py --verify +git diff --check +python3 scripts/fast_build.py fast-build --changed-only codex-cli codex-tui echo "=== Merge Complete ===" ``` diff --git a/justfile b/justfile index f2b6a769125..b55d08cb8ef 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,7 @@ set working-directory := "codex-rs" set positional-arguments +python_cmd := if os_family() == "windows" { "py -3" } else { "python3" } +build_script := "../scripts/fast_build.py" # Display help help: @@ -86,11 +88,11 @@ write-app-server-schema *args: log *args: if [ "${1:-}" = "--" ]; then shift; fi; cargo run -p codex-state --bin logs_client -- "$@" +fast-build *args: + {{python_cmd}} {{build_script}} fast-build --changed-only "$@" -[no-cd] -configure-remotes: - ./scripts/sync-upstream.sh --configure-only +fast-build-install *args: + {{python_cmd}} {{build_script}} fast-build-install --changed-only "$@" -[no-cd] -sync-upstream *args: - ./scripts/sync-upstream.sh "$@" +upstream-sync *args: + {{python_cmd}} {{build_script}} upstream-sync "$@" diff --git a/scripts/fast_build.py b/scripts/fast_build.py new file mode 100755 index 00000000000..38f69e6a804 --- /dev/null +++ b/scripts/fast_build.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKSPACE_ROOT = REPO_ROOT / "codex-rs" +CACHE_PATH = REPO_ROOT / ".codex-fast-build-cache.json" +DEFAULT_JOBS = int(os.environ.get("CODEX_FAST_BUILD_JOBS", os.environ.get("FAST_BUILD_JOBS", "6"))) +DEFAULT_METHOD = os.environ.get("CODEX_FAST_BUILD_METHOD", "md5") +_CARGO_METADATA_DIGEST: str | None = None +DEFAULT_PROFILE = os.environ.get("CODEX_FAST_BUILD_PROFILE", "release") + + +def binary_name(base: str) -> str: + return f"{base}.exe" if os.name == "nt" else base + + +def npm_command(script: str) -> list[str]: + return ["npm.cmd", "run", script] if os.name == "nt" else ["npm", "run", script] + + +@dataclass(frozen=True) +class Target: + name: str + kind: str + cwd: Path + build_cmd: list[str] + watch_roots: tuple[Path, ...] + install_map: dict[Path, str] = field(default_factory=dict) + process_names: tuple[str, ...] = () + package: str | None = None + description: str = "" + + +def now() -> str: + return datetime.now(timezone.utc).astimezone().strftime("%H:%M:%S") + + +class Logger: + def __init__(self, log_file: Path | None) -> None: + self.log_file = log_file + if self.log_file: + self.log_file.parent.mkdir(parents=True, exist_ok=True) + + def emit(self, level: str, message: str) -> None: + line = f"[{now()}] [{level}] {message}" + print(line) + if self.log_file: + with self.log_file.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + + def info(self, message: str) -> None: + self.emit("INFO", message) + + def warn(self, message: str) -> None: + self.emit("WARN", message) + + def error(self, message: str) -> None: + self.emit("ERROR", message) + + +TARGETS: dict[str, Target] = { + "codex-cli": Target( + name="codex-cli", + kind="rust", + cwd=WORKSPACE_ROOT, + build_cmd=["cargo", "build", "--release", "-p", "codex-cli", "--features", "custom-features"], + watch_roots=(WORKSPACE_ROOT / "cli", WORKSPACE_ROOT / "core", WORKSPACE_ROOT / "exec", WORKSPACE_ROOT / "protocol", WORKSPACE_ROOT / "config", WORKSPACE_ROOT / "state", WORKSPACE_ROOT / "mcp-server", WORKSPACE_ROOT / "deep-research", WORKSPACE_ROOT / "utils"), + install_map={WORKSPACE_ROOT / "target" / "release" / binary_name("codex"): binary_name("codex")}, + process_names=("codex",), + package="codex-cli", + description="Rust CLI binary", + ), + "codex-tui": Target( + name="codex-tui", + kind="rust", + cwd=WORKSPACE_ROOT, + build_cmd=["cargo", "build", "--release", "-p", "codex-tui"], + watch_roots=(WORKSPACE_ROOT / "tui", WORKSPACE_ROOT / "core", WORKSPACE_ROOT / "protocol", WORKSPACE_ROOT / "state", WORKSPACE_ROOT / "utils"), + install_map={WORKSPACE_ROOT / "target" / "release" / binary_name("codex-tui"): binary_name("codex-tui")}, + process_names=("codex-tui",), + package="codex-tui", + description="Rust TUI binary", + ), + "codex-gui": Target( + name="codex-gui", + kind="rust", + cwd=WORKSPACE_ROOT / "gui", + build_cmd=["cargo", "build", "--release", "--manifest-path", str((WORKSPACE_ROOT / "gui" / "Cargo.toml").resolve())], + watch_roots=(WORKSPACE_ROOT / "gui", WORKSPACE_ROOT / "core", WORKSPACE_ROOT / "protocol", WORKSPACE_ROOT / "state"), + install_map={WORKSPACE_ROOT / "gui" / "target" / "release" / binary_name("codex-gui"): binary_name("codex-gui")}, + process_names=("codex-gui",), + description="Custom Rust GUI binary", + ), + "codex-gui-x": Target( + name="codex-gui-x", + kind="node", + cwd=REPO_ROOT / "codex-gui-x", + build_cmd=npm_command("build"), + watch_roots=(REPO_ROOT / "codex-gui-x" / "src", REPO_ROOT / "codex-gui-x" / "public", REPO_ROOT / "codex-gui-x" / "package.json", REPO_ROOT / "codex-gui-x" / "tsconfig.json", REPO_ROOT / "codex-gui-x" / "vite.config.ts"), + description="Custom Vite GUI bundle", + ), + "extensions": Target( + name="extensions", + kind="node", + cwd=REPO_ROOT / "extensions", + build_cmd=npm_command("compile"), + watch_roots=(REPO_ROOT / "extensions" / "src", REPO_ROOT / "extensions" / "package.json", REPO_ROOT / "extensions" / "tsconfig.json"), + description="Custom extension bundle", + ), +} + +SHARED_WATCH = [ + REPO_ROOT / "justfile", + WORKSPACE_ROOT / "justfile", + WORKSPACE_ROOT / "Cargo.toml", + WORKSPACE_ROOT / "Cargo.lock", + REPO_ROOT / "package.json", +] + + +def iter_files(paths: Iterable[Path]) -> list[Path]: + files: set[Path] = set() + ignored_dirs = {"target", "node_modules", ".git", "dist", "build", ".next", ".turbo", ".cache"} + ignored_suffixes = {".pyc", ".pyo", ".pkl", ".log", ".tmp", ".swp"} + for path in paths: + if not path.exists(): + continue + if path.is_file(): + files.add(path) + continue + for child in path.rglob("*"): + if any(part in ignored_dirs for part in child.parts): + continue + if child.is_file() and child.suffix not in ignored_suffixes: + files.add(child) + return sorted(files) + + +def cargo_metadata_digest() -> str: + global _CARGO_METADATA_DIGEST + if _CARGO_METADATA_DIGEST is None: + completed = subprocess.run(["cargo", "metadata", "--no-deps", "--format-version", "1"], cwd=WORKSPACE_ROOT, capture_output=True, text=True, check=True) + _CARGO_METADATA_DIGEST = hashlib.md5(completed.stdout.encode("utf-8")).hexdigest() + return _CARGO_METADATA_DIGEST + + +def fingerprint(target: Target, files: list[Path], method: str) -> str: + digest = hashlib.md5() + for path in files: + relative = path.relative_to(REPO_ROOT).as_posix() + digest.update(relative.encode("utf-8")) + stat = path.stat() + if method == "mtime": + digest.update(f"{stat.st_mtime_ns}:{stat.st_size}".encode("utf-8")) + continue + if method == "cargo-metadata": + digest.update(f"{stat.st_mtime_ns}:{stat.st_size}:{path.parent.name}".encode("utf-8")) + continue + digest.update(path.read_bytes()) + if method == "cargo-metadata" and target.kind == "rust": + digest.update(cargo_metadata_digest().encode("utf-8")) + return digest.hexdigest() + + +def load_cache() -> dict: + if not CACHE_PATH.exists(): + return {"targets": {}} + try: + return json.loads(CACHE_PATH.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {"targets": {}} + + +def save_cache(cache: dict) -> None: + CACHE_PATH.write_text(json.dumps(cache, indent=2, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8") + + +def detect_changed(target_names: list[str], method: str, logger: Logger) -> tuple[list[str], dict]: + cache = load_cache() + cache.setdefault("targets", {}) + changed: list[str] = [] + for name in target_names: + target = TARGETS[name] + files = iter_files([*target.watch_roots, *SHARED_WATCH]) + current = fingerprint(target, files, method) + record = cache["targets"].get(name, {}) + previous = record.get("fingerprint") if record.get("method") == method else None + logger.info(f"{name}: scanned {len(files)} input files via {method}") + if current != previous: + changed.append(name) + cache["targets"][name] = { + "description": target.description, + "fingerprint": current, + "files": [path.relative_to(REPO_ROOT).as_posix() for path in files], + "method": method, + "updated_at": datetime.now(timezone.utc).isoformat(), + } + return changed, cache + + +def run(command: list[str], cwd: Path, logger: Logger, extra_env: dict[str, str]) -> None: + display_cwd = cwd.relative_to(REPO_ROOT) if cwd.is_relative_to(REPO_ROOT) else cwd + logger.info(f"Running in {display_cwd}: {' '.join(command)}") + env = os.environ.copy() + env.update(extra_env) + subprocess.run(command, cwd=cwd, env=env, check=True) + + +def cargo_bin_dir() -> Path: + cargo_home = Path(os.environ.get("CARGO_HOME", Path.home() / ".cargo")) + return cargo_home / "bin" + + +def kill_processes(targets: Iterable[Target], logger: Logger) -> None: + names = sorted({name for target in targets for name in target.process_names}) + if not names: + return + logger.info(f"Stopping running processes: {', '.join(names)}") + for name in names: + if os.name == "nt": + subprocess.run(["taskkill", "/F", "/IM", binary_name(name)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + else: + subprocess.run(["pkill", "-f", name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + + +def install_targets(target_names: list[str], logger: Logger) -> None: + install_dir = cargo_bin_dir() + install_dir.mkdir(parents=True, exist_ok=True) + chosen = [TARGETS[name] for name in target_names] + kill_processes(chosen, logger) + for target in chosen: + for source, dest_name in target.install_map.items(): + if not source.exists(): + raise FileNotFoundError(f"Missing build artifact for {target.name}: {source}") + destination = install_dir / dest_name + shutil.copy2(source, destination) + logger.info(f"Installed {target.name}: {source.relative_to(REPO_ROOT)} -> {destination}") + + +def resolve_targets(requested: list[str] | None) -> list[str]: + if not requested: + return ["codex-cli", "codex-tui", "codex-gui", "codex-gui-x", "extensions"] + resolved: list[str] = [] + for item in requested: + for part in [part.strip() for part in item.split(",") if part.strip()]: + if part == "all": + resolved.extend(TARGETS) + continue + if part not in TARGETS: + raise SystemExit(f"Unknown target '{part}'. Choose from: {', '.join(sorted(TARGETS))}") + resolved.append(part) + return list(dict.fromkeys(resolved)) + + +def add_common_flags(command: list[str], args: argparse.Namespace) -> list[str]: + if command[:2] != ["cargo", "build"]: + return command + patched = command.copy() + if "--release" in patched and args.profile != "release": + patched.remove("--release") + patched[2:2] = ["--profile", args.profile] + patched.extend(["-j", str(args.jobs)]) + return patched + + +def cmd_list(_: argparse.Namespace) -> int: + for target in TARGETS.values(): + print(f"{target.name:12} {target.kind:5} {target.description}") + return 0 + + +def cmd_fast_build(args: argparse.Namespace) -> int: + logger = Logger(args.log_file) + target_names = resolve_targets(args.targets) + changed, cache = detect_changed(target_names, args.method, logger) + selected = target_names if args.force or not args.changed_only else changed + if args.changed_only and not changed and not args.force: + logger.info("No target inputs changed; skipping build.") + save_cache(cache) + return 0 + env = { + "CARGO_BUILD_JOBS": str(args.jobs), + "CODEX_FAST_BUILD_JOBS": str(args.jobs), + "CARGO_INCREMENTAL": "1", + } + if args.deny_warnings: + env["RUSTFLAGS"] = "-D warnings" + for name in selected: + command = add_common_flags(TARGETS[name].build_cmd, args) + run(command, TARGETS[name].cwd, logger, env) + save_cache(cache) + return 0 + + +def cmd_fast_build_install(args: argparse.Namespace) -> int: + result = cmd_fast_build(args) + if result != 0: + return result + logger = Logger(args.log_file) + installable = [name for name in resolve_targets(args.targets) if TARGETS[name].install_map] + if not installable: + logger.info("Selected targets do not produce installable binaries; skipping install.") + return 0 + install_targets(installable, logger) + return 0 + + +def cmd_upstream_sync(args: argparse.Namespace) -> int: + logger = Logger(args.log_file) + if not args.no_fetch: + run(["git", "fetch", args.remote], REPO_ROOT, logger, {}) + merge_target = f"{args.remote}/{args.branch}" + merge = subprocess.run(["git", "merge", "--no-commit", "--no-ff", merge_target], cwd=REPO_ROOT, text=True) + if merge.returncode not in {0, 1}: + raise subprocess.CalledProcessError(merge.returncode, merge.args) + conflicts = subprocess.run(["git", "diff", "--name-only", "--diff-filter=U"], cwd=REPO_ROOT, capture_output=True, text=True, check=True) + paths = [line.strip() for line in conflicts.stdout.splitlines() if line.strip()] + if paths: + resolver = [sys.executable, str(REPO_ROOT / "scripts" / "resolve_merge_conflicts.py"), *paths] + if args.rule: + for rule in args.rule: + resolver.extend(["--rule", rule]) + run(resolver, REPO_ROOT, logger, {}) + subprocess.run(["git", "add", *paths], cwd=REPO_ROOT, check=True) + logger.info("Upstream sync flow complete. Review the working tree before committing.") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Codex fast-build and upstream-sync orchestration") + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_common(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument("targets", nargs="*", help="Target names or comma-separated groups (default: core + custom GUI/extensions)") + subparser.add_argument("--jobs", type=int, default=DEFAULT_JOBS, help="Parallel build jobs (default: 6 or CODEX_FAST_BUILD_JOBS)") + subparser.add_argument("--method", choices=["md5", "mtime", "cargo-metadata"], default=DEFAULT_METHOD, help="Change detection mode") + subparser.add_argument("--profile", default=DEFAULT_PROFILE, help="Cargo profile name (default: release)") + subparser.add_argument("--force", action="store_true", help="Build all selected targets regardless of cache") + subparser.add_argument("--changed-only", action="store_true", help="Only build targets whose inputs changed") + subparser.add_argument("--no-deny-warnings", dest="deny_warnings", action="store_false", help="Do not inject RUSTFLAGS=-D warnings") + subparser.add_argument("--log-file", type=Path, help="Append log output to a file") + subparser.set_defaults(deny_warnings=True) + + list_parser = subparsers.add_parser("list-targets", help="List supported build targets") + list_parser.set_defaults(func=cmd_list) + + fast_build = subparsers.add_parser("fast-build", help="Run differential builds for selected targets") + add_common(fast_build) + fast_build.set_defaults(func=cmd_fast_build) + + fast_build_install = subparsers.add_parser("fast-build-install", help="Build changed targets and install produced binaries") + add_common(fast_build_install) + fast_build_install.set_defaults(func=cmd_fast_build_install) + + upstream_sync = subparsers.add_parser("upstream-sync", help="Fetch/merge upstream and auto-resolve conflicts") + upstream_sync.add_argument("--remote", default="upstream", help="Upstream remote name") + upstream_sync.add_argument("--branch", default="main", help="Upstream branch name") + upstream_sync.add_argument("--no-fetch", action="store_true", help="Skip git fetch before merge") + upstream_sync.add_argument("--rule", action="append", help="Additional resolver rule in glob=strategy form") + upstream_sync.add_argument("--log-file", type=Path, help="Append log output to a file") + upstream_sync.set_defaults(func=cmd_upstream_sync) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/resolve_merge_conflicts.py b/scripts/resolve_merge_conflicts.py new file mode 100755 index 00000000000..db12c677b24 --- /dev/null +++ b/scripts/resolve_merge_conflicts.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import fnmatch +from dataclasses import dataclass +from pathlib import Path +import re + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class Rule: + pattern: str + strategy: str + + +DEFAULT_RULES = [ + Rule("codex-rs/**", "upstream-reinject"), + Rule("codex-rs/gui/**", "upstream-reinject"), + Rule("codex-gui-x/**", "custom"), + Rule("extensions/**", "custom"), + Rule("docs/**", "upstream"), + Rule("CHANGELOG.md", "upstream-reinject"), + Rule("justfile", "upstream-reinject"), + Rule("codex-rs/justfile", "upstream-reinject"), +] + +CONFLICT_RE = re.compile( + r"^<<<<<<< .*?\n(?P.*?)^=======\n(?P.*?)^>>>>>>> .*?$", + re.MULTILINE | re.DOTALL, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Resolve merge conflicts with upstream-first reinjection rules") + parser.add_argument("paths", nargs="+", help="Conflicted paths to resolve") + parser.add_argument("--rule", action="append", default=[], help="Extra rule in glob=strategy form") + return parser.parse_args() + + +def load_rules(extra: list[str]) -> list[Rule]: + rules = DEFAULT_RULES.copy() + for entry in extra: + pattern, _, strategy = entry.partition("=") + if not pattern or not strategy: + raise SystemExit(f"Invalid rule '{entry}'. Expected glob=strategy") + rules.insert(0, Rule(pattern, strategy)) + return rules + + +def choose_strategy(path: str, rules: list[Rule]) -> str: + for rule in rules: + if fnmatch.fnmatch(path, rule.pattern): + return rule.strategy + return "upstream-reinject" + + +def unique_local_lines(local: str, upstream: str) -> list[str]: + upstream_lines = {line.rstrip() for line in upstream.splitlines() if line.strip()} + seen: set[str] = set() + kept: list[str] = [] + for line in local.splitlines(): + stripped = line.rstrip() + if not stripped or stripped in upstream_lines or stripped in seen: + continue + seen.add(stripped) + kept.append(line) + return kept + + +def resolve_block(local: str, upstream: str, strategy: str) -> str: + if strategy == "custom": + return local + if strategy == "upstream": + return upstream + reinjected = unique_local_lines(local, upstream) + if not reinjected: + return upstream + body = upstream.rstrip("\n") + trailer = "\n" if body else "" + trailer += "\n".join(reinjected) + if not trailer.endswith("\n"): + trailer += "\n" + return body + trailer + + +def resolve_file(path: Path, rules: list[Rule]) -> None: + text = path.read_text(encoding="utf-8") + relative = path.relative_to(REPO_ROOT).as_posix() + strategy = choose_strategy(relative, rules) + + def repl(match: re.Match[str]) -> str: + return resolve_block(match.group("local"), match.group("upstream"), strategy) + + updated, count = CONFLICT_RE.subn(repl, text) + if count == 0: + print(f"skip {relative}: no merge markers found") + return + path.write_text(updated, encoding="utf-8") + print(f"resolved {relative}: strategy={strategy}, blocks={count}") + + +def main() -> int: + args = parse_args() + rules = load_rules(args.rule) + for raw_path in args.paths: + path = (REPO_ROOT / raw_path).resolve() + if not path.exists(): + raise SystemExit(f"Missing conflicted file: {raw_path}") + resolve_file(path, rules) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/upstream_sync.py b/scripts/upstream_sync.py new file mode 100755 index 00000000000..5d54fb4c059 --- /dev/null +++ b/scripts/upstream_sync.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +FAST_BUILD = REPO_ROOT / "scripts" / "fast_build.py" + + +def main() -> int: + command = [sys.executable, str(FAST_BUILD), "upstream-sync", *sys.argv[1:]] + completed = subprocess.run(command, cwd=REPO_ROOT) + return completed.returncode + + +if __name__ == "__main__": + raise SystemExit(main())