RFC: Extend omnigraph upgrade with explicit storage migrations

.github/branch-protection.json

@@ -1,5 +1,5 @@
 {
-  "_comment": "Branch protection policy for main. Applied via scripts/apply-branch-protection.sh. See docs/dev/branch-protection.md for rationale. CODEOWNERS was removed (2-person team where both maintainers own everything, so code-owner review added friction without value). Review is no longer code-owner-scoped and no approvals are required; the reporting PR checks below are the gate. Graph Vocabulary Guard currently reports a successful PR skip while its full audit runs post-merge, on tags, and by manual dispatch. The full workspace suite, the format fence, and the RustFS S3 suite run on pull requests as reporting contexts and again after merge; the Azurite suite runs post-merge; immutable workflow refs are checked on pull requests. A red main branch is stop-the-line until fixed or reverted.",
+  "_comment": "Branch protection policy for main. Applied via scripts/apply-branch-protection.sh. See docs/dev/branch-protection.md for rationale. CODEOWNERS was removed (2-person team where both maintainers own everything, so code-owner review added friction without value). Review is no longer code-owner-scoped and no approvals are required; the reporting PR checks below are the gate. Graph Vocabulary Guard currently reports a successful PR skip while its full audit runs post-merge, on tags, and by manual dispatch. The full workspace suite, the format fence, and the RustFS S3 suite run on pull requests as reporting contexts and again after merge; the Azurite suite runs post-merge; immutable workflow refs are checked on pull requests. A red main branch is stop-the-line until fixed or reverted. Storage Upgrade Compatibility is a required context on every change, including documentation-only pull requests.",
   "required_status_checks": {
     "strict": true,
     "contexts": [
@@ -11,7 +11,8 @@
       "Format (rustfmt)",
       "Lint (clippy)",
       "GQ Logic Tests",
-      "Fix Regression Gate"
+      "Fix Regression Gate",
+      "Storage Upgrade Compatibility"
     ]
   },
   "enforce_admins": false,

.github/workflows/ci.yml

@@ -161,6 +161,9 @@ jobs:
       - name: Enforce release vocabulary gates
         run: python3 scripts/check-release-vocabulary-gates.py
 
+      - name: Check required storage upgrade coverage
+        run: python3 scripts/check-storage-upgrade-ci.py --self-test
+
   azure_contract_guards:
     name: Azure Contract Guards
     runs-on: ubuntu-latest
@@ -602,6 +605,86 @@ jobs:
             || { echo "::error::exact v6 format fence did not pass"; exit 1; }
 
 
+
+  storage_upgrade_compatibility:
+    name: Storage Upgrade Compatibility
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+    env:
+      CARGO_TERM_COLOR: always
+      RUST_MIN_STACK: 16777216
+      OMNIGRAPH_REQUIRE_STORAGE_UPGRADE_TESTS: '1'
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler libprotobuf-dev
+
+      - name: Install pinned toolchain
+        run: rustup toolchain install
+
+      - name: Cache Rust build data
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          workspaces: |
+            . -> target
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-on-failure: true
+
+      - name: Check storage compatibility gate configuration
+        run: python3 scripts/check-storage-upgrade-ci.py --self-test
+
+      - name: Install genuine v0.9 and v0.10 migration predecessors
+        run: |
+          set -euo pipefail
+          v09_dir="$RUNNER_TEMP/omnigraph-v09"
+          v6_dir="$RUNNER_TEMP/omnigraph-v010"
+          REPO_SLUG=ModernRelay/omnigraph VERSION=v0.9.0 INSTALL_DIR="$v09_dir" \
+            bash scripts/install.sh
+          REPO_SLUG=ModernRelay/omnigraph VERSION=v0.10.0 INSTALL_DIR="$v6_dir" \
+            bash scripts/install.sh
+          test -x "$v09_dir/omnigraph"
+          test -x "$v6_dir/omnigraph"
+          [[ "$("$v09_dir/omnigraph" --version)" == "omnigraph 0.9.0" ]] \
+            || { echo "::error::storage upgrade requires the genuine v0.9.0 CLI"; exit 1; }
+          [[ "$("$v6_dir/omnigraph" --version)" == "omnigraph 0.10.0" ]] \
+            || { echo "::error::storage upgrade requires the genuine v0.10.0 CLI"; exit 1; }
+          echo "OMNIGRAPH_V09_BIN=$v09_dir/omnigraph" >> "$GITHUB_ENV"
+          echo "OMNIGRAPH_V6_BIN=$v6_dir/omnigraph" >> "$GITHUB_ENV"
+
+      - name: Run required storage upgrade crossversion tests
+        run: |
+          set -euo pipefail
+          test_log="$RUNNER_TEMP/storage-upgrade-crossversion.log"
+          cargo test --workspace --locked --test crossversion_upgrade --features "$FAILPOINT_FEATURES" storage_upgrade -- --test-threads=1 2>&1 | tee "$test_log"
+          python3 scripts/check-storage-upgrade-ci.py --check-log crossversion "$test_log"
+
+      - name: Run required storage upgrade engine tests
+        run: |
+          set -euo pipefail
+          test_log="$RUNNER_TEMP/storage-upgrade-engine.log"
+          cargo test --locked -p omnigraph-engine --lib --features failpoints db::manifest::upgrade::tests -- --test-threads=1 2>&1 | tee "$test_log"
+          python3 scripts/check-storage-upgrade-ci.py --check-log engine "$test_log"
+
+      - name: Run required storage upgrade lance tests
+        run: |
+          set -euo pipefail
+          test_log="$RUNNER_TEMP/storage-upgrade-lance.log"
+          cargo test --locked -p omnigraph-engine --test lance_version_columns --features failpoints -- --test-threads=1 2>&1 | tee "$test_log"
+          python3 scripts/check-storage-upgrade-ci.py --check-log lance "$test_log"
+
+      - name: Run required storage upgrade protocol tests
+        run: |
+          set -euo pipefail
+          test_log="$RUNNER_TEMP/storage-upgrade-protocol.log"
+          cargo test --locked -p omnigraph-engine --test forbidden_apis --features failpoints -- --test-threads=1 2>&1 | tee "$test_log"
+          python3 scripts/check-storage-upgrade-ci.py --check-log protocol "$test_log"
+
   v5_v7_format_fence:
     name: V5 ↔ V7 Format Fence
     needs: classify_changes

crates/omnigraph-cli/src/cli.rs

@@ -17,7 +17,7 @@ COMMANDS BY CAPABILITY:\n  \
 any — run against a graph, served (--server / --profile) or embedded (--store / a \
 URI): query, mutate, load, blob, branch, snapshot, export, commit, changes, schema show/apply.\n  \
 served — require a server: graphs (registry scope).\n  \
-direct — direct storage access; reject --server (init, optimize, rebuild-full-text-indexes, \
+direct — direct storage access; reject --server (init, upgrade, optimize, rebuild-full-text-indexes, \
 repair, cleanup, schema plan, lint).\n  \
 control — manage or inspect a cluster (cluster via --config; policy & queries via \
 --cluster).\n  \
@@ -295,6 +295,19 @@ pub(crate) enum Command {
         #[arg(long)]
         force: bool,
     },
+    /// Upgrade graph storage offline using registered migration handlers
+    Upgrade {
+        /// Standalone graph storage URI; alternatively use --store
+        uri: Option<String>,
+        /// Run read-only preflight without conversion or recovery writes
+        #[arg(long)]
+        check: bool,
+        /// Requested storage format (defaults to the binary's declared target)
+        #[arg(long, value_name = "N")]
+        to_format: Option<u32>,
+        #[arg(long)]
+        json: bool,
+    },
     /// Compact small Lance fragments in every backing dataset of the graph
     Optimize {
         /// Graph URI

crates/omnigraph-cli/src/main.rs

@@ -55,6 +55,7 @@ mod managed_http_fixture;
 mod output;
 mod planes;
 mod scope;
+mod upgrade;
 use cli::*;
 use helpers::*;
 use output::*;
@@ -1357,6 +1358,23 @@ async fn main() -> Result<()> {
                 }
             }
         }
+        Command::Upgrade {
+            uri,
+            check,
+            to_format,
+            json,
+        } => {
+            upgrade::run(
+                &cli.profile,
+                &cli.store,
+                uri,
+                check,
+                to_format,
+                json,
+                cli.quiet,
+            )
+            .await?;
+        }
         Command::Optimize { uri, json } => {
             let uri = resolve_maintenance_uri(
                 cli.profile.as_deref(),

crates/omnigraph-cli/src/planes.rs

@@ -265,6 +265,7 @@ pub(crate) fn command_plane(cmd: &Command) -> Plane {
         Command::Queries { .. } => Plane::Control,
         Command::Policy { .. } => Plane::Control,
         Command::Init { .. }
+        | Command::Upgrade { .. }
         | Command::Optimize { .. }
         | Command::RebuildFullTextIndexes { .. }
         | Command::Repair { .. }
@@ -320,6 +321,7 @@ pub(crate) fn command_label(cmd: &Command) -> &'static str {
         Command::Mutate { .. } => "mutate",
         Command::Alias { .. } => "alias",
         Command::Policy { .. } => "policy",
+        Command::Upgrade { .. } => "upgrade",
         Command::Optimize { .. } => "optimize",
         Command::RebuildFullTextIndexes { .. } => "rebuild-full-text-indexes",
         Command::Repair { .. } => "repair",

crates/omnigraph-cli/src/upgrade.rs

@@ -0,0 +1,197 @@
+use super::*;
+
+pub(crate) async fn run(
+    profile: &Option<String>,
+    store: &Option<String>,
+    uri: Option<String>,
+    check: bool,
+    to_format: Option<u32>,
+    json: bool,
+    quiet: bool,
+) -> Result<()> {
+    let target = scope::resolve_scope(
+        &operator::load_operator_config()?,
+        planes::Capability::Direct,
+        scope::ScopeFlags {
+            profile: profile.as_deref(),
+            store: store.as_deref(),
+            server: None,
+            cluster: None,
+            graph: None,
+            uri,
+        },
+    )?;
+    if target.cluster.is_some() {
+        bail!(
+            "upgrade refuses cluster-managed graphs; a qualified cluster upgrade operation is required"
+        );
+    }
+    let uri = resolve_local_uri(target.uri, "upgrade")?;
+    let uri = omnigraph::storage::normalize_root_uri(&uri)?;
+    let uri = if omnigraph::storage::storage_kind_for_uri(&uri)?
+        == omnigraph::storage::StorageKind::Local
+    {
+        std::fs::canonicalize(&uri)?
+            .to_str()
+            .ok_or_else(|| color_eyre::eyre::eyre!("upgrade path is not valid UTF-8"))?
+            .to_owned()
+    } else {
+        uri
+    };
+    if let Some(root) = omnigraph_cluster::cluster_root_for_graph_uri(&uri)
+        .await
+        .map_err(|diagnostic| {
+            color_eyre::eyre::eyre!("{}: {}", diagnostic.path, diagnostic.message)
+        })?
+    {
+        bail!(
+            "upgrade refuses graph `{uri}` inside cluster `{root}`; a qualified cluster upgrade operation is required"
+        );
+    }
+    if !check {
+        echo_write_target(quiet, "upgrade", &uri, false);
+    }
+    let report =
+        omnigraph::db::upgrade_storage(&uri, omnigraph::db::UpgradeOptions { check, to_format })
+            .await?;
+    if json {
+        print_json(&report)?;
+    } else {
+        print_human(&report)?;
+    }
+    if !report.success() {
+        std::process::exit(1);
+    }
+    Ok(())
+}
+
+fn print_human(report: &omnigraph::db::UpgradeReport) -> Result<()> {
+    let mode = serde_json::to_value(report.mode)?;
+    let outcome = serde_json::to_value(report.outcome)?;
+    println!(
+        "upgrade {}: {} ({})",
+        report.location,
+        outcome.as_str().unwrap_or("unknown"),
+        mode.as_str().unwrap_or("unknown")
+    );
+    println!(
+        "graph identity: {}",
+        report.graph_identity.as_deref().unwrap_or("unknown")
+    );
+    println!(
+        "format: {} -> {}{}",
+        report
+            .observed_format
+            .map_or_else(|| "unknown".into(), |v| v.to_string()),
+        report.target_format,
+        if report.target_defaulted {
+            " (default target)"
+        } else {
+            ""
+        }
+    );
+    println!("route: {}", report.route.join(" -> "));
+    println!(
+        "completed handlers: {}",
+        report.completed_handlers.join(", ")
+    );
+    println!(
+        "last durable completed boundary: {}",
+        report
+            .last_durable_completed_boundary
+            .as_deref()
+            .unwrap_or("unknown")
+    );
+    println!(
+        "work: {} metadata rows, {} retained snapshots, {} payload bytes copied, {} payload bytes rewritten",
+        report.work.metadata_rows,
+        report.work.retained_snapshots,
+        report.work.payload_bytes_copied,
+        report.work.payload_bytes_rewritten
+    );
+    println!(
+        "validation bytes: {}",
+        report
+            .work
+            .validation_bytes
+            .map_or_else(|| "unknown".into(), |v| v.to_string())
+    );
+    for exclusion in &report.work.external_blob_exclusions {
+        println!("external bytes excluded from preservation: {exclusion}");
+    }
+    for property in &report.work.historical_blob_identity_limits {
+        println!(
+            "historical Blob delivery retains the pre-0.10 property-lifetime restriction: {property}"
+        );
+    }
+    for finding in &report.findings {
+        println!("{}: {}", finding.code, finding.message);
+    }
+    if let Some(recovery) = &report.recovery {
+        println!("failed handler: {}", recovery.failed_handler);
+        println!("recovery executable: {}", recovery.executable_compatibility);
+        println!("recovery action: {}", recovery.action);
+    }
+    if matches!(report.mode, omnigraph::db::UpgradeMode::Check) {
+        println!(
+            "Check is advisory. Stop all writers and maintenance and retain a verified backup before execution."
+        );
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn storage_upgrade_parses_explicit_route_and_check_options() {
+        let cli = Cli::try_parse_from([
+            "omnigraph",
+            "upgrade",
+            "graph.omni",
+            "--check",
+            "--to-format",
+            "7",
+            "--json",
+        ])
+        .unwrap();
+        assert!(
+            matches!(&cli.command, Command::Upgrade { uri: Some(uri), check: true, to_format: Some(7), json: true } if uri == "graph.omni")
+        );
+        assert_eq!(
+            planes::command_capability(&cli.command),
+            planes::Capability::Direct
+        );
+        assert!(planes::guard_addressing(&cli).is_ok());
+        let cli = Cli::try_parse_from(["omnigraph", "--store", "graph.omni", "upgrade"]).unwrap();
+        assert!(matches!(
+            cli.command,
+            Command::Upgrade {
+                uri: None,
+                check: false,
+                to_format: None,
+                json: false
+            }
+        ));
+    }
+
+    #[test]
+    fn storage_upgrade_rejects_served_and_cluster_addressing() {
+        for flag in ["--server", "--cluster", "--graph"] {
+            let cli =
+                Cli::try_parse_from(["omnigraph", flag, "prod", "upgrade", "graph.omni"]).unwrap();
+            assert!(planes::guard_addressing(&cli).is_err(), "{flag}");
+        }
+        assert!(
+            Cli::try_parse_from([
+                "omnigraph",
+                "upgrade",
+                "graph.omni",
+                "--to-format",
+                "not-a-version"
+            ])
+            .is_err()
+        );
+    }
+}